From 90d8d207eb674c971d41d0862ab132c597af6130 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Feb 2018 17:43:22 +0100 Subject: [PATCH 001/265] Added geocoding and routing services --- .../cartodb_services/tomtom/__init__.py | 2 + .../cartodb_services/tomtom/geocoder.py | 87 ++++++++++ .../cartodb_services/tomtom/routing.py | 148 ++++++++++++++++++ .../cartodb_services/test/credentials.py | 3 + .../test/test_tomtomgeocoder.py | 56 +++++++ .../test/test_tomtomrouting.py | 52 ++++++ 6 files changed, 348 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py create mode 100644 server/lib/python/cartodb_services/test/test_tomtomgeocoder.py create mode 100644 server/lib/python/cartodb_services/test/test_tomtomrouting.py diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py new file mode 100644 index 0000000..c318187 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py @@ -0,0 +1,2 @@ +from geocoder import TomTomGeocoder +from routing import TomTomRouting, TomTomRoutingResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py new file mode 100644 index 0000000..2bd0525 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -0,0 +1,87 @@ +import json +import requests +from uritemplate import URITemplate +from cartodb_services.metrics import Traceable +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools.qps import qps_retry +from cartodb_services.tools.normalize import normalize + +BASEURI = ('https://api.tomtom.com/search/2/geocode/' + '{searchtext}.JSON' + '?key={apiKey}' + '&limit=1') +ENTRY_RESULTS = 'results' +ENTRY_POSITION = 'position' +ENTRY_LON = 'lon' +ENTRY_LAT = 'lat' + + +class TomTomGeocoder(Traceable): + ''' + Python wrapper for the TomTom Geocoder service. + ''' + + def __init__(self, apikey, logger, service_params=None): + service_params = service_params or {} + self._apikey = apikey + self._logger = logger + + def _uri(self, searchtext, countries=None): + baseuri = BASEURI + '&countrySet={}'.format(countries) \ + if countries else BASEURI + uri = URITemplate(baseuri).expand(apiKey=self._apikey, + searchtext=searchtext.encode('utf-8')) + return uri + + def _parse_geocoder_response(self, response): + json_response = json.loads(response) + + if json_response and json_response[ENTRY_RESULTS]: + result = json_response[ENTRY_RESULTS][0] + return self._extract_lng_lat_from_feature(result) + else: + return [] + + def _extract_lng_lat_from_feature(self, result): + position = result[ENTRY_POSITION] + longitude = position[ENTRY_LON] + latitude = position[ENTRY_LAT] + return [longitude, latitude] + + @qps_retry(qps=5) + def geocode(self, searchtext, city=None, state_province=None, + country=None): + if searchtext and searchtext.strip(): + address = [normalize(searchtext)] + if city: + address.append(normalize(city)) + if state_province: + address.append(normalize(state_province)) + else: + return [] + + uri = self._uri(searchtext=', '.join(address), countries=country) + + try: + response = requests.get(uri) + + if response.status_code == requests.codes.ok: + return self._parse_geocoder_response(response.text) + elif response.status_code == requests.codes.bad_request: + return [] + elif response.status_code == requests.codes.unprocessable_entity: + return [] + else: + raise ServiceException(response.status_code, response) + except requests.Timeout as te: + # In case of timeout we want to stop the job because the server + # could be down + self._logger.error('Timeout connecting to TomTom geocoding server', + te) + raise ServiceException('Error geocoding {0} using TomTom'.format( + searchtext), None) + except requests.ConnectionError as ce: + # Don't raise the exception to continue with the geocoding job + self._logger.error('Error connecting to TomTom geocoding server', + exception=ce) + return [] diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py new file mode 100644 index 0000000..ec10eb0 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -0,0 +1,148 @@ +''' +Python client for the TomTom Routing service. +''' + +import json +import requests +from uritemplate import URITemplate +from cartodb_services.metrics import Traceable +from cartodb_services.tools import PolyLine +from cartodb_services.tools.coordinates import (validate_coordinates, + marshall_coordinates) +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools.qps import qps_retry + +BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/' + '{coordinates}' + '/json' + '?key={apikey}' + '&travelMode={travelmode}' + '&computeBestOrder=true') + +NUM_WAYPOINTS_MIN = 2 +NUM_WAYPOINTS_MAX = 20 + +PROFILE_DRIVING = 'car' +PROFILE_CYCLING = 'bicycle' +PROFILE_WALKING = 'pedestrian' +DEFAULT_PROFILE = PROFILE_DRIVING + +VALID_PROFILES = [PROFILE_DRIVING, + PROFILE_CYCLING, + PROFILE_WALKING] + +ENTRY_ROUTES = 'routes' +ENTRY_SUMMARY = 'summary' +ENTRY_LENGTH = 'lengthInMeters' +ENTRY_TIME = 'travelTimeInSeconds' +ENTRY_LEGS = 'legs' +ENTRY_POINTS = 'points' +ENTRY_LATITUDE = 'latitude' +ENTRY_LONGITUDE = 'longitude' + + +class TomTomRouting(Traceable): + ''' + Python wrapper for the TomTom Routing service. + ''' + + def __init__(self, apikey, logger, service_params=None): + service_params = service_params or {} + self._apikey = apikey + self._logger = logger + + def _uri(self, coordinates, profile=DEFAULT_PROFILE): + uri = URITemplate(BASEURI).expand(apikey=self._apikey, + coordinates=coordinates, + travelmode=profile) + return uri + + def _validate_profile(self, profile): + if profile not in VALID_PROFILES: + raise ValueError('{profile} is not a valid profile. ' + 'Valid profiles are: {valid_profiles}'.format( + profile=profile, + valid_profiles=', '.join( + [x for x in VALID_PROFILES]))) + + def _marshall_coordinates(self, coordinates): + return ':'.join(['{lat},{lon}'.format(lat=coordinate.latitude, + lon=coordinate.longitude) + for coordinate in coordinates]) + + def _parse_routing_response(self, response): + json_response = json.loads(response) + + if json_response: + route = json_response[ENTRY_ROUTES][0] # Force the first route + + geometry = self._parse_legs(route[ENTRY_LEGS]) + summary = route[ENTRY_SUMMARY] + distance = summary[ENTRY_LENGTH] + duration = summary[ENTRY_TIME] + + return TomTomRoutingResponse(geometry, distance, duration) + else: + return TomTomRoutingResponse(None, None, None) + + def _parse_legs(self, legs): + geometry = [] + for leg in legs: + points = leg[ENTRY_POINTS] + for point in points: + geometry.append((point[ENTRY_LATITUDE], + point[ENTRY_LONGITUDE])) + return geometry + + @qps_retry(qps=5) + def directions(self, waypoints, profile=DEFAULT_PROFILE): + self._validate_profile(profile) + validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX) + + coordinates = self._marshall_coordinates(waypoints) + + uri = self._uri(coordinates, profile) + + try: + response = requests.get(uri) + + if response.status_code == requests.codes.ok: + return self._parse_routing_response(response.text) + elif response.status_code == requests.codes.bad_request: + return TomTomRoutingResponse(None, None, None) + elif response.status_code == requests.codes.unprocessable_entity: + return TomTomRoutingResponse(None, None, None) + else: + raise ServiceException(response.status_code, response) + except requests.Timeout as te: + # In case of timeout we want to stop the job because the server + # could be down + self._logger.error('Timeout connecting to TomTom routing service', + te) + raise ServiceException('Error getting routing data from TomTom', + None) + except requests.ConnectionError as ce: + # Don't raise the exception to continue with the geocoding job + self._logger.error('Error connecting to TomTom routing service', + exception=ce) + return TomTomRoutingResponse(None, None, None) + + +class TomTomRoutingResponse: + + def __init__(self, shape, length, duration): + self._shape = shape + self._length = length + self._duration = duration + + @property + def shape(self): + return self._shape + + @property + def length(self): + return self._length + + @property + def duration(self): + return self._duration diff --git a/server/lib/python/cartodb_services/test/credentials.py b/server/lib/python/cartodb_services/test/credentials.py index 5f10182..c818902 100644 --- a/server/lib/python/cartodb_services/test/credentials.py +++ b/server/lib/python/cartodb_services/test/credentials.py @@ -4,3 +4,6 @@ def mapbox_api_key(): """Returns Mapbox API key. Requires setting MAPBOX_API_KEY environment variable.""" return os.environ['MAPBOX_API_KEY'] +def tomtom_api_key(): + """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable.""" + return os.environ['TOMTOM_API_KEY'] diff --git a/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py new file mode 100644 index 0000000..706b7de --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py @@ -0,0 +1,56 @@ +import unittest +from mock import Mock +from cartodb_services.tomtom import TomTomGeocoder +from cartodb_services.tools.exceptions import ServiceException +from credentials import tomtom_api_key + +INVALID_APIKEY = 'invalid_apikey' +VALID_ADDRESS = 'Plaza Mayor 3, Valladolid' +WELL_KNOWN_LONGITUDE = -4.728 +WELL_KNOWN_LATITUDE = 41.653 + + +class TomTomGeocoderTestCase(unittest.TestCase): + def setUp(self): + self.geocoder = TomTomGeocoder(apikey=tomtom_api_key(), logger=Mock()) + + def test_invalid_token(self): + invalid_geocoder = TomTomGeocoder(apikey=INVALID_APIKEY, logger=Mock()) + with self.assertRaises(ServiceException): + invalid_geocoder.geocode(VALID_ADDRESS) + + def test_valid_request(self): + place = self.geocoder.geocode(VALID_ADDRESS) + + self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE) + self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE) + + def test_valid_request_namedplace(self): + place = self.geocoder.geocode(searchtext='Barcelona') + + assert place + + def test_valid_request_namedplace2(self): + place = self.geocoder.geocode(searchtext='New York', country='us') + + assert place + + def test_odd_characters(self): + place = self.geocoder.geocode(searchtext='Barcelona; "Spain"') + + assert place + + def test_empty_request(self): + place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None) + + assert place == [] + + def test_empty_search_text_request(self): + place = self.geocoder.geocode(searchtext=' ', country='us', city=None, state_province="") + + assert place == [] + + def test_unknown_place_request(self): + place = self.geocoder.geocode(searchtext='[unknown]', country='ch', state_province=None, city=None) + + assert place == [] diff --git a/server/lib/python/cartodb_services/test/test_tomtomrouting.py b/server/lib/python/cartodb_services/test/test_tomtomrouting.py new file mode 100644 index 0000000..fab18c4 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_tomtomrouting.py @@ -0,0 +1,52 @@ +import unittest +from mock import Mock +from cartodb_services.tomtom import TomTomRouting +from cartodb_services.tomtom.routing import DEFAULT_PROFILE +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools import Coordinate +from credentials import tomtom_api_key + +INVALID_APIKEY = 'invalid_apikey' +VALID_WAYPOINTS = [Coordinate(13.42936, 52.50931), + Coordinate(13.43872, 52.50274)] +NUM_WAYPOINTS_MAX = 20 +INVALID_WAYPOINTS_EMPTY = [] +INVALID_WAYPOINTS_MIN = [Coordinate(13.42936, 52.50931)] +INVALID_WAYPOINTS_MAX = [Coordinate(13.42936, 52.50931) + for x in range(0, NUM_WAYPOINTS_MAX + 2)] +VALID_PROFILE = DEFAULT_PROFILE +INVALID_PROFILE = 'invalid_profile' + + +class TomTomRoutingTestCase(unittest.TestCase): + def setUp(self): + self.routing = TomTomRouting(apikey=tomtom_api_key(), logger=Mock()) + + def test_invalid_profile(self): + with self.assertRaises(ValueError): + self.routing.directions(VALID_WAYPOINTS, INVALID_PROFILE) + + def test_invalid_waypoints_empty(self): + with self.assertRaises(ValueError): + self.routing.directions(INVALID_WAYPOINTS_EMPTY, VALID_PROFILE) + + def test_invalid_waypoints_min(self): + with self.assertRaises(ValueError): + self.routing.directions(INVALID_WAYPOINTS_MIN, VALID_PROFILE) + + def test_invalid_waypoints_max(self): + with self.assertRaises(ValueError): + self.routing.directions(INVALID_WAYPOINTS_MAX, VALID_PROFILE) + + def test_invalid_token(self): + invalid_routing = TomTomRouting(apikey=INVALID_APIKEY, logger=Mock()) + with self.assertRaises(ServiceException): + invalid_routing.directions(VALID_WAYPOINTS, + VALID_PROFILE) + + def test_valid_request(self): + route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE) + + assert route.shape + assert route.length + assert route.duration From 03e1d1ca613b590494602a19bed606c4bf2f2b7c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 21 Feb 2018 19:13:59 +0100 Subject: [PATCH 002/265] Change default provider to mapbox instead of mapzen --- .../cartodb_services/cartodb_services/metrics/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index aab4dd5..f068ab2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -136,7 +136,7 @@ class RoutingConfig(ServiceConfig): ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' - DEFAULT_PROVIDER = MAPZEN_PROVIDER + DEFAULT_PROVIDER = MAPBOX_PROVIDER QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' METRICS_LOG_KEY = 'routing_log_path' @@ -226,7 +226,7 @@ class IsolinesRoutingConfig(ServiceConfig): MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' HEREMAPS_PROVIDER = 'heremaps' - DEFAULT_PROVIDER = MAPZEN_PROVIDER + DEFAULT_PROVIDER = MAPBOX_PROVIDER METRICS_LOG_KEY = 'isolines_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -391,7 +391,7 @@ class GeocoderConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' - DEFAULT_PROVIDER = MAPZEN_GEOCODER + DEFAULT_PROVIDER = MAPBOX_GEOCODER METRICS_LOG_KEY = 'geocoder_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None): From 39dabffb8529993d1d6b13164a57a77463bdbab6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 21 Feb 2018 19:14:29 +0100 Subject: [PATCH 003/265] Now is not mandatory to have mapzen configuration becuase its deprecated as provider --- .../python/cartodb_services/cartodb_services/metrics/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index f068ab2..82fff8d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -589,8 +589,9 @@ class ServicesDBConfig: def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') + # We dont use mapzen anymore so we don't need to check for its configuration if not mapzen_conf_json: - raise ConfigException('Mapzen configuration missing') + return else: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] From e3f23adfdd38cadd7e926d02966c47d5c311c957 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 21 Feb 2018 19:14:47 +0100 Subject: [PATCH 004/265] Bump to version 0.17.0 --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 901ee87..a87deb4 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.16.7', + version='0.17.0', description='CartoDB Services API Python Library', From 199788748b8713d97c13507e3b038cd8468aaf9e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 21 Feb 2018 19:14:56 +0100 Subject: [PATCH 005/265] Updated NEWS.md --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 250eaeb..9fff051 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +February 22th, 2018 +================== +* Version `0.17.0` of the python library + * Change default provider to Mapbox + * Remove the obligatory nature of the Mapzen configuration due to its deprecation as provider February 13th, 2018 ================== From cbc19b869c8c13c93210a7ec7642c8e41423e1c3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 22 Feb 2018 11:14:33 +0100 Subject: [PATCH 006/265] Fix CR suggestions --- .../cartodb_services/metrics/config.py | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 82fff8d..16e076d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -576,51 +576,51 @@ class ServicesDBConfig: heremaps_conf_json = self._get_conf('heremaps_conf') if not heremaps_conf_json: raise ConfigException('Here maps configuration missing') - else: - heremaps_conf = json.loads(heremaps_conf_json) - self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id'] - self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] - self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ - 'geocoder_cost_per_hit'] - self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {}) - self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] - self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] - self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {}) + + heremaps_conf = json.loads(heremaps_conf_json) + self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id'] + self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] + self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ + 'geocoder_cost_per_hit'] + self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {}) + self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] + self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] + self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {}) def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') # We dont use mapzen anymore so we don't need to check for its configuration if not mapzen_conf_json: return - else: - mapzen_conf = json.loads(mapzen_conf_json) - self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] - self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] - self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {}) - self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {}) - self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] - self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] - self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {}) - self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] - self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] - self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) + + mapzen_conf = json.loads(mapzen_conf_json) + self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] + self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] + self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {}) + self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {}) + self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] + self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] + self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {}) + self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] + self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] + self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) def _get_mapbox_config(self): mapbox_conf_json = self._get_conf('mapbox_conf') if not mapbox_conf_json: raise ConfigException('Mapbox configuration missing') - else: - mapbox_conf = json.loads(mapbox_conf_json) - self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys'] - self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota'] - self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {}) - self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {}) - self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys'] - self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota'] - self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {}) - self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys'] - self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] - self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) + + mapbox_conf = json.loads(mapbox_conf_json) + self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys'] + self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota'] + self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {}) + self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {}) + self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys'] + self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota'] + self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {}) + self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys'] + self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] + self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') From 05e2cc981e243d556eb78f97abdf9ffda072b48d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 22 Feb 2018 11:20:16 +0100 Subject: [PATCH 007/265] Change min log level for tests to WARNING --- client/test/expected/00_installation_test.out | 2 ++ client/test/sql/00_installation_test.sql | 2 ++ server/extension/test/expected/00_install_test.out | 2 ++ server/extension/test/sql/00_install_test.sql | 2 ++ 4 files changed, 8 insertions(+) diff --git a/client/test/expected/00_installation_test.out b/client/test/expected/00_installation_test.out index fe179b7..650b141 100644 --- a/client/test/expected/00_installation_test.out +++ b/client/test/expected/00_installation_test.out @@ -1,3 +1,5 @@ +-- Only show warning or error messages in the tests output +SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; CREATE EXTENSION plpythonu; diff --git a/client/test/sql/00_installation_test.sql b/client/test/sql/00_installation_test.sql index f091cd4..49eab80 100644 --- a/client/test/sql/00_installation_test.sql +++ b/client/test/sql/00_installation_test.sql @@ -1,3 +1,5 @@ +-- Only show warning or error messages in the tests output +SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; CREATE EXTENSION plpythonu; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index f87ca05..cfb9ee7 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -1,3 +1,5 @@ +-- Only show warning or error messages in the tests output +SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; CREATE EXTENSION plpythonu; diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 9ffe2e1..2d7d41a 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -1,3 +1,5 @@ +-- Only show warning or error messages in the tests output +SET client_min_messages TO WARNING; -- Install dependencies CREATE EXTENSION postgis; CREATE EXTENSION plpythonu; From 177c19f9357995ca824f756efa53dec4793a4caf Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 23 Feb 2018 15:59:54 +0100 Subject: [PATCH 008/265] Added TomTom isolines --- .../cartodb_services/tomtom/__init__.py | 1 + .../cartodb_services/tomtom/isolines.py | 141 ++++++++++++++++++ .../cartodb_services/tomtom/routing.py | 22 ++- .../cartodb_services/tomtom/types.py | 16 ++ .../test/test_tomtomisoline.py | 33 ++++ 5 files changed, 200 insertions(+), 13 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/types.py create mode 100644 server/lib/python/cartodb_services/test/test_tomtomisoline.py diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py index c318187..8c5df3c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py @@ -1,2 +1,3 @@ from geocoder import TomTomGeocoder from routing import TomTomRouting, TomTomRoutingResponse +from isolines import TomTomIsolines, TomTomIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py new file mode 100644 index 0000000..83bbfe1 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py @@ -0,0 +1,141 @@ +''' +Python implementation for TomTom services based isolines. +''' + +import json +import requests +from uritemplate import URITemplate + +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools.qps import qps_retry +from cartodb_services.tools import Coordinate +from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT, + MAX_SPEEDS) + +BASEURI = ('https://api.tomtom.com/routing/1/calculateReachableRange/' + '{origin}' + '/json' + '?key={apikey}' + '&timeBudgetInSec={time}' + '&travelMode={travelmode}' + '&departAt={departat}') + +ENTRY_REACHABLERANGE = 'reachableRange' +ENTRY_BOUNDARY = 'boundary' +ENTRY_LATITUDE = 'latitude' +ENTRY_LONGITUDE = 'longitude' + + +class TomTomIsolines(): + ''' + Python wrapper for TomTom services based isolines. + ''' + + def __init__(self, apikey, logger, service_params=None): + service_params = service_params or {} + self._apikey = apikey + self._logger = logger + + def _uri(self, origin, time_range, profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): + uri = URITemplate(BASEURI).expand(apikey=self._apikey, + origin=origin, + time=time_range, + travelmode=profile, + departat=date_time) + return uri + + def _validate_profile(self, profile): + if profile not in VALID_PROFILES: + raise ValueError('{profile} is not a valid profile. ' + 'Valid profiles are: {valid_profiles}'.format( + profile=profile, + valid_profiles=', '.join( + [x for x in VALID_PROFILES]))) + + def _parse_coordinates(self, boundary): + return [Coordinate(c[ENTRY_LONGITUDE], c[ENTRY_LONGITUDE]) for c in boundary] + + def _parse_reachablerange_response(self, response): + json_response = json.loads(response) + + if json_response: + reachable_range = json_response[ENTRY_REACHABLERANGE] + + return self._parse_coordinates(reachable_range[ENTRY_BOUNDARY]) + + @qps_retry(qps=5) + def _calculate_isoline(self, origin, time_range, + profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): + origin = '{lat},{lon}'.format(lat=origin.latitude, + lon=origin.longitude) + + uri = self._uri(origin, time_range, profile, date_time) + try: + response = requests.get(uri) + + if response.status_code == requests.codes.ok: + return self._parse_reachablerange_response(response.text) + elif response.status_code == requests.codes.bad_request: + return [] + elif response.status_code == requests.codes.unprocessable_entity: + return [] + else: + raise ServiceException(response.status_code, response) + except requests.Timeout as te: + # In case of timeout we want to stop the job because the server + # could be down + self._logger.error('Timeout connecting to TomTom calculateReachableRange service', + te) + raise ServiceException('Error getting calculateReachableRange data from TomTom', + None) + except requests.ConnectionError as ce: + # Don't raise the exception to continue with the geocoding job + self._logger.error('Error connecting to TomTom calculateReachableRange service', + exception=ce) + return [] + + def calculate_isochrone(self, origin, time_ranges, + profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): + self._validate_profile(profile) + + isochrones = [] + for time_range in time_ranges: + coordinates = self._calculate_isoline(origin=origin, + time_range=time_range, + profile=profile, + date_time=date_time) + + isochrones.append(TomTomIsochronesResponse(coordinates, + time_range)) + return isochrones + + def calculate_isodistance(self, origin, distance_range, + profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): + self._validate_profile(profile) + + max_speed = MAX_SPEEDS[profile] + time_range = distance_range / max_speed + + return self._calculate_isoline(origin=origin, + time_range=time_range, + profile=profile, + date_time=date_time) + + +class TomTomIsochronesResponse: + + def __init__(self, coordinates, duration): + self._coordinates = coordinates + self._duration = duration + + @property + def coordinates(self): + return self._coordinates + + @property + def duration(self): + return self._duration diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index ec10eb0..dfc30fb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -11,26 +11,19 @@ from cartodb_services.tools.coordinates import (validate_coordinates, marshall_coordinates) from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry +from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT) BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/' '{coordinates}' '/json' '?key={apikey}' '&travelMode={travelmode}' + '&departAt={departat}' '&computeBestOrder=true') NUM_WAYPOINTS_MIN = 2 NUM_WAYPOINTS_MAX = 20 -PROFILE_DRIVING = 'car' -PROFILE_CYCLING = 'bicycle' -PROFILE_WALKING = 'pedestrian' -DEFAULT_PROFILE = PROFILE_DRIVING - -VALID_PROFILES = [PROFILE_DRIVING, - PROFILE_CYCLING, - PROFILE_WALKING] - ENTRY_ROUTES = 'routes' ENTRY_SUMMARY = 'summary' ENTRY_LENGTH = 'lengthInMeters' @@ -51,10 +44,12 @@ class TomTomRouting(Traceable): self._apikey = apikey self._logger = logger - def _uri(self, coordinates, profile=DEFAULT_PROFILE): + def _uri(self, coordinates, profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): uri = URITemplate(BASEURI).expand(apikey=self._apikey, coordinates=coordinates, - travelmode=profile) + travelmode=profile, + departat=date_time) return uri def _validate_profile(self, profile): @@ -95,13 +90,14 @@ class TomTomRouting(Traceable): return geometry @qps_retry(qps=5) - def directions(self, waypoints, profile=DEFAULT_PROFILE): + def directions(self, waypoints, profile=DEFAULT_PROFILE, + date_time=DEFAULT_DEPARTAT): self._validate_profile(profile) validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX) coordinates = self._marshall_coordinates(waypoints) - uri = self._uri(coordinates, profile) + uri = self._uri(coordinates, profile, date_time) try: response = requests.get(uri) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py new file mode 100644 index 0000000..63f0b86 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -0,0 +1,16 @@ +PROFILE_DRIVING = 'car' +PROFILE_CYCLING = 'bicycle' +PROFILE_WALKING = 'pedestrian' +DEFAULT_PROFILE = PROFILE_DRIVING + +DEFAULT_DEPARTAT = 'now' + +VALID_PROFILES = [PROFILE_DRIVING, + PROFILE_CYCLING, + PROFILE_WALKING] + +MAX_SPEEDS = { + PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed + PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed + PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed +} diff --git a/server/lib/python/cartodb_services/test/test_tomtomisoline.py b/server/lib/python/cartodb_services/test/test_tomtomisoline.py new file mode 100644 index 0000000..8fdf901 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_tomtomisoline.py @@ -0,0 +1,33 @@ +import unittest +from mock import Mock +from cartodb_services.tomtom.isolines import TomTomIsolines, DEFAULT_PROFILE +from cartodb_services.tools import Coordinate + +from credentials import tomtom_api_key + +VALID_ORIGIN = Coordinate(-73.989, 40.733) + + +class TomTomIsolinesTestCase(unittest.TestCase): + + def setUp(self): + self.tomtom_isolines = TomTomIsolines(apikey=tomtom_api_key(), + logger=Mock()) + + def test_calculate_isochrone(self): + time_ranges = [300, 900] + solution = self.tomtom_isolines.calculate_isochrone( + origin=VALID_ORIGIN, + profile=DEFAULT_PROFILE, + time_ranges=time_ranges) + + assert solution + + def test_calculate_isodistance(self): + distance_range = 10000 + solution = self.tomtom_isolines.calculate_isodistance( + origin=VALID_ORIGIN, + profile=DEFAULT_PROFILE, + distance_range=distance_range) + + assert solution From bc4c9fea33dd4c39e89f3676c3a63268c65ce279 Mon Sep 17 00:00:00 2001 From: Antonio Date: Mon, 26 Feb 2018 12:45:08 +0100 Subject: [PATCH 009/265] Using Mapbox returned destinations as coordinates for isolines --- .../cartodb_services/mapbox/isolines.py | 24 +++++++++++++------ .../cartodb_services/mapbox/matrix_client.py | 2 ++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py index e7f06c1..b837a3b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py @@ -4,6 +4,7 @@ Uses the Mapbox Time Matrix service. ''' import json +from cartodb_services.tools import Coordinate from cartodb_services.tools.spherical import (get_angles, calculate_dest_location) from cartodb_services.mapbox.matrix_client import (validate_profile, @@ -11,7 +12,9 @@ from cartodb_services.mapbox.matrix_client import (validate_profile, PROFILE_WALKING, PROFILE_DRIVING, PROFILE_CYCLING, - ENTRY_DURATIONS) + ENTRY_DURATIONS, + ENTRY_DESTINATIONS, + ENTRY_LOCATION) MAX_SPEEDS = { PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed @@ -53,6 +56,7 @@ class MapboxIsolines(): return [] costs = [None] * number_of_angles + destinations = [None] * number_of_angles for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]): if cost: @@ -60,7 +64,11 @@ class MapboxIsolines(): else: costs[idx] = isorange - return costs + for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]): + destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0], + destination[ENTRY_LOCATION][1]) + + return costs, destinations def calculate_isochrone(self, origin, time_ranges, profile=DEFAULT_PROFILE): @@ -122,10 +130,12 @@ class MapboxIsolines(): # NOTE: sometimes it cannot calculate the cost and returns None. # Just assume isorange and stop the calculations there - costs = cost_method(origin=origin, targets=location_estimates, - isorange=isorange, profile=profile, - unit_factor=unit_factor, - number_of_angles=number_of_angles) + costs, destinations = cost_method(origin=origin, + targets=location_estimates, + isorange=isorange, + profile=profile, + unit_factor=unit_factor, + number_of_angles=number_of_angles) if not costs: continue @@ -153,7 +163,7 @@ class MapboxIsolines(): location_estimates_filtered = [] for i, c in enumerate(costs): if c != isorange: - location_estimates_filtered.append(location_estimates[i]) + location_estimates_filtered.append(destinations[i]) return location_estimates_filtered diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py index 8d547b6..fd4cf1f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py @@ -30,6 +30,8 @@ VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC, PROFILE_WALKING] ENTRY_DURATIONS = 'durations' +ENTRY_DESTINATIONS = 'destinations' +ENTRY_LOCATION = 'location' def validate_profile(profile): From 97f1611d6252cc96f009ec67adb4ed1761ff389e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Feb 2018 18:09:21 +0100 Subject: [PATCH 010/265] Only get mapzen/mapbox values in case the provider of routing is one of these --- .../cartodb_services/metrics/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 16e076d..6179d1b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -147,10 +147,12 @@ class RoutingConfig(ServiceConfig): self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY] if not self._routing_provider: self._routing_provider = self.DEFAULT_PROVIDER - self._mapzen_api_key = self._db_config.mapzen_routing_api_key - self._mapzen_service_params = self._db_config.mapzen_routing_service_params - self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys - self._mapbox_service_params = self._db_config.mapbox_routing_service_params + if self._routing_provider == self.MAPZEN_PROVIDER: + self._mapzen_api_key = self._db_config.mapzen_routing_api_key + self._mapzen_service_params = self._db_config.mapzen_routing_service_params + elif self._routing_provider == self.MAPBOX_PROVIDER: + self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys + self._mapbox_service_params = self._db_config.mapbox_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) From ad46de1156ef9db4f0763a0b0b538542aa0827ca Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Feb 2018 18:11:50 +0100 Subject: [PATCH 011/265] Bump version and update NEWS.md --- NEWS.md | 6 ++++++ server/lib/python/cartodb_services/setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 9fff051..036318d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ + +February 27th, 2018 +================== +* Version `0.17.1` of the python library + * Fix bug when the mapzen credentials are not in the db config and we keep getting them + February 22th, 2018 ================== * Version `0.17.0` of the python library diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index a87deb4..95a2c16 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.0', + version='0.17.1', description='CartoDB Services API Python Library', From 9fb04fdc241618523e1dc4991b2c324f3b6ef105 Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 28 Feb 2018 11:49:23 +0100 Subject: [PATCH 012/265] Version bumped --- NEWS.md | 5 +++++ server/lib/python/cartodb_services/setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 036318d..53c6bab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,9 @@ +February 22th, 2018 +================== +* Version `0.17.2` of the python library + * Fix bug with Mapbox isolines not stopping at the seacoast + February 27th, 2018 ================== * Version `0.17.1` of the python library diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 95a2c16..c1c5893 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.1', + version='0.17.2', description='CartoDB Services API Python Library', From 4308b5f351aef2339a2166a12b02ecc74e58d1d7 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 10:23:58 +0100 Subject: [PATCH 013/265] Remove too far away points (filtering by time) --- .../python/cartodb_services/cartodb_services/mapbox/isolines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py index b837a3b..9c1eb07 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py @@ -162,7 +162,7 @@ class MapboxIsolines(): # delete points that got None location_estimates_filtered = [] for i, c in enumerate(costs): - if c != isorange: + if c < isorange * (1 + tolerance): location_estimates_filtered.append(destinations[i]) return location_estimates_filtered From 35f743164e6c01c7a10fa77baeed1f7379fc2a2e Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 10:27:37 +0100 Subject: [PATCH 014/265] Keep deleting points that got None --- .../python/cartodb_services/cartodb_services/mapbox/isolines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py index 9c1eb07..4d6563f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py @@ -162,7 +162,7 @@ class MapboxIsolines(): # delete points that got None location_estimates_filtered = [] for i, c in enumerate(costs): - if c < isorange * (1 + tolerance): + if c != isorange and c < isorange * (1 + tolerance): location_estimates_filtered.append(destinations[i]) return location_estimates_filtered From 596189185f3a2f48106eaa1ef2c7311db610cc04 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 16:01:22 +0100 Subject: [PATCH 015/265] Added configuration for TomTom services --- README.md | 9 ++ server/extension/test/sql/00_install_test.sql | 1 + .../cartodb_services/metrics/config.py | 111 +++++++++++++++ .../cartodb_services/metrics/quota.py | 6 + .../cartodb_services/metrics/user.py | 8 +- .../service/tomtom_geocoder_config.py | 128 +++++++++++++++++ .../service/tomtom_isolines_config.py | 123 ++++++++++++++++ .../refactor/service/tomtom_routing_config.py | 131 ++++++++++++++++++ .../cartodb_services/tomtom/types.py | 4 + .../cartodb_services/test/credentials.py | 2 + .../test/metrics/test_config.py | 16 ++- .../cartodb_services/test/test_helper.py | 1 + 12 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py diff --git a/README.md b/README.md index e5b0d32..1e9bee7 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,15 @@ SELECT CDB_Conf_SetConf( ); ``` +#### TomTom configuration + +```sql +SELECT CDB_Conf_SetConf( + 'tomtom_conf', + '{"routing": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "isolines": {"api_keys": ["your_api_key"], "monthly_quota": 1500000}}' +); +``` + #### Data Observatory ```sql diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 9ffe2e1..4ce2667 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -14,6 +14,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); +SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index aab4dd5..cf4ac96 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -136,6 +136,7 @@ class RoutingConfig(ServiceConfig): ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' + TOMTOM_PROVIDER = 'tomtom' DEFAULT_PROVIDER = MAPZEN_PROVIDER QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' @@ -151,6 +152,8 @@ class RoutingConfig(ServiceConfig): self._mapzen_service_params = self._db_config.mapzen_routing_service_params self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys self._mapbox_service_params = self._db_config.mapbox_routing_service_params + self._tomtom_api_keys = self._db_config.tomtom_routing_api_keys + self._tomtom_service_params = self._db_config.tomtom_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -161,6 +164,8 @@ class RoutingConfig(ServiceConfig): return 'routing_mapzen' elif self._routing_provider == self.MAPBOX_PROVIDER: return 'routing_mapbox' + elif self._routing_provider == self.TOMTOM_PROVIDER: + return 'routing_tomtom' @property def provider(self): @@ -190,6 +195,18 @@ class RoutingConfig(ServiceConfig): def mapbox_service_params(self): return self._mapbox_service_params + @property + def tomtom_provider(self): + return self._routing_provider == self.TOMTOM_PROVIDER + + @property + def tomtom_api_keys(self): + return self._tomtom_api_keys + + @property + def tomtom_service_params(self): + return self._tomtom_service_params + @property def monthly_quota(self): return self._monthly_quota @@ -225,6 +242,7 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_PROVIDER_KEY = 'geocoder_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' + TOMTOM_PROVIDER = 'tomtom' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = MAPZEN_PROVIDER METRICS_LOG_KEY = 'isolines_log_path' @@ -258,6 +276,9 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params + elif self._isolines_provider == self.TOMTOM_PROVIDER: + self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys + self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params @property def service_type(self): @@ -267,6 +288,8 @@ class IsolinesRoutingConfig(ServiceConfig): return 'mapzen_isolines' elif self._isolines_provider == self.MAPBOX_PROVIDER: return 'mapbox_isolines' + elif self._isolines_provider == self.TOMTOM_PROVIDER: + return 'tomtom_isolines' @property def google_services_user(self): @@ -328,6 +351,18 @@ class IsolinesRoutingConfig(ServiceConfig): def mapbox_provider(self): return self._isolines_provider == self.MAPBOX_PROVIDER + @property + def tomtom_isolines_api_keys(self): + return self._tomtom_isolines_api_keys + + @property + def tomtom_isolines_service_params(self): + return self._tomtom_isolines_service_params + + @property + def tomtom_provider(self): + return self._isolines_provider == self.TOMTOM_PROVIDER + @property def heremaps_provider(self): return self._isolines_provider == self.HEREMAPS_PROVIDER @@ -386,6 +421,8 @@ class GeocoderConfig(ServiceConfig): GEOCODER_PROVIDER = 'geocoder_provider' MAPBOX_GEOCODER = 'mapbox' MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys' + TOMTOM_GEOCODER = 'tomtom' + TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' USERNAME_KEY = 'username' @@ -415,6 +452,9 @@ class GeocoderConfig(ServiceConfig): elif self._geocoder_provider == self.MAPBOX_GEOCODER: if not self.mapbox_api_keys: raise ConfigException("""Mapbox config is not set up""") + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + if not self.tomtom_api_keys: + raise ConfigException("""TomTom config is not set up""") return True @@ -450,6 +490,10 @@ class GeocoderConfig(ServiceConfig): self._mapbox_api_keys = db_config.mapbox_geocoder_api_keys self._cost_per_hit = 0 self._mapbox_service_params = db_config.mapbox_geocoder_service_params + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys + self._cost_per_hit = 0 + self._tomtom_service_params = db_config.tomtom_geocoder_service_params @property def service_type(self): @@ -459,6 +503,8 @@ class GeocoderConfig(ServiceConfig): return 'geocoder_mapzen' elif self._geocoder_provider == self.MAPBOX_GEOCODER: return 'geocoder_mapbox' + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + return 'geocoder_tomtom' elif self._geocoder_provider == self.NOKIA_GEOCODER: return 'geocoder_here' @@ -478,6 +524,10 @@ class GeocoderConfig(ServiceConfig): def mapbox_geocoder(self): return self._geocoder_provider == self.MAPBOX_GEOCODER + @property + def tomtom_geocoder(self): + return self._geocoder_provider == self.TOMTOM_GEOCODER + @property def google_client_id(self): return self._google_maps_client_id @@ -529,6 +579,14 @@ class GeocoderConfig(ServiceConfig): def mapbox_service_params(self): return self._mapbox_service_params + @property + def tomtom_api_keys(self): + return self._tomtom_api_keys + + @property + def tomtom_service_params(self): + return self._tomtom_service_params + @property def is_high_resolution(self): return True @@ -559,6 +617,7 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_mapbox_config() + self._get_tomtom_config() self._get_data_observatory_config() def _get_server_config(self): @@ -621,6 +680,22 @@ class ServicesDBConfig: self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) + def _get_tomtom_config(self): + tomtom_conf_json = self._get_conf('tomtom_conf') + if not tomtom_conf_json: + raise ConfigException('TomTom configuration missing') + else: + tomtom_conf = json.loads(tomtom_conf_json) + self._tomtom_isolines_api_keys = tomtom_conf['isolines']['api_keys'] + self._tomtom_isolines_quota = tomtom_conf['isolines']['monthly_quota'] + self._tomtom_isolines_service_params = tomtom_conf.get('isolines', {}).get('service', {}) + self._tomtom_routing_api_keys = tomtom_conf['routing']['api_keys'] + self._tomtom_routing_quota = tomtom_conf['routing']['monthly_quota'] + self._tomtom_routing_service_params = tomtom_conf['routing'].get('service', {}) + self._tomtom_geocoder_api_keys = tomtom_conf['geocoder']['api_keys'] + self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota'] + self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {}) + def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') if not do_conf_json: @@ -754,6 +829,42 @@ class ServicesDBConfig: def mapbox_geocoder_service_params(self): return self._mapbox_geocoder_service_params + @property + def tomtom_isolines_api_keys(self): + return self._tomtom_isolines_api_keys + + @property + def tomtom_isolines_monthly_quota(self): + return self._tomtom_isolines_quota + + @property + def tomtom_isolines_service_params(self): + return self._tomtom_isolines_service_params + + @property + def tomtom_routing_api_keys(self): + return self._tomtom_routing_api_keys + + @property + def tomtom_routing_monthly_quota(self): + return self._tomtom_routing_quota + + @property + def tomtom_routing_service_params(self): + return self._tomtom_routing_service_params + + @property + def tomtom_geocoder_api_keys(self): + return self._tomtom_geocoder_api_keys + + @property + def tomtom_geocoder_monthly_quota(self): + return self._tomtom_geocoder_quota + + @property + def tomtom_geocoder_service_params(self): + return self._tomtom_geocoder_service_params + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 7be4c6c..62c489a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -81,12 +81,18 @@ class QuotaChecker: elif re.match('mapbox_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('tomtom_isolines', + self._user_service_config.service_type) is not None: + return self.__check_isolines_quota() elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() elif re.match('routing_mapbox', self._user_service_config.service_type) is not None: return self.__check_routing_quota() + elif re.match('routing_tomtom', + self._user_service_config.service_type) is not None: + return self.__check_routing_quota() elif re.match('obs_*', self._user_service_config.service_type) is not None: return self.__check_data_observatory_quota() diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 3c0163e..70bc9a3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -22,8 +22,10 @@ class UserMetricsService: SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' + SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_MAPBOX_ROUTING = 'routing_mapbox' + SERVICE_TOMTOM_ROUTING = 'routing_tomtom' SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' @@ -36,10 +38,12 @@ class UserMetricsService: def used_quota(self, service_type, date): if service_type in [self.SERVICE_HERE_ISOLINES, self.SERVICE_MAPZEN_ISOLINES, - self.SERVICE_MAPBOX_ISOLINES]: + self.SERVICE_MAPBOX_ISOLINES, + self.SERVICE_TOMTOM_ISOLINES]: return self.__used_isolines_quota(service_type, date) elif service_type in [self.SERVICE_MAPZEN_ROUTING, - self.SERVICE_MAPBOX_ROUTING]: + self.SERVICE_MAPBOX_ROUTING, + self.SERVICE_TOMTOM_ROUTING]: return self.__used_routing_quota(service_type, date) elif service_type == self.SERVICE_OBSERVATORY: return self.__used_observatory_quota(service_type, date) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py new file mode 100644 index 0000000..b987e82 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py @@ -0,0 +1,128 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_GEOCODER_APIKEY_ROUNDROBIN + + +class TomTomGeocoderConfig(object): + """ + Configuration needed to operate the TomTom geocoder service. + """ + + def __init__(self, + geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._geocoding_quota = geocoding_quota + self._soft_geocoding_limit = soft_geocoding_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'geocoder_tomtom' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_GEOCODER_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + # TODO: for BW compat, remove + @property + def google_geocoder(self): + return False + + +class TomTomGeocoderConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['geocoder']['api_keys'] + tomtom_service_params = tomtom_server_conf['geocoder'].get('service', {}) + + geocoding_quota = self._get_quota() + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('geocoder_log_path', None) + + return TomTomGeocoderConfig(geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') + if geocoding_quota is '': + return 0 + + return int(geocoding_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py new file mode 100644 index 0000000..1750be1 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py @@ -0,0 +1,123 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_ISOLINES_APIKEY_ROUNDROBIN + + +class TomTomIsolinesConfig(object): + """ + Configuration needed to operate the TomTom directions service. + """ + + def __init__(self, + isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._isolines_quota = isolines_quota + self._soft_isolines_limit = soft_isolines_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'tomtom_isolines' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return self._soft_isolines_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_ISOLINES_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class TomTomIsolinesConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['isolines']['api_keys'] + tomtom_service_params = tomtom_server_conf['isolines'].get('service', {}) + + isolines_quota = self._get_quota() + soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('isolines_log_path', None) + + return TomTomIsolinesConfig(isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota') + if isolines_quota is '': + return 0 + + return int(isolines_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py new file mode 100644 index 0000000..2c9ab9c --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py @@ -0,0 +1,131 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_ROUTING_APIKEY_ROUNDROBIN + + +class TomTomRoutingConfig(object): + """ + Configuration needed to operate the TomTom directions service. + """ + + def __init__(self, + routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._routing_quota = routing_quota + self._soft_routing_limit = soft_routing_limit + self._monthly_quota = monthly_quota + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'routing_tomtom' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def routing_quota(self): + return self._routing_quota + + @property + def soft_limit(self): + return self._soft_routing_limit + + @property + def monthly_quota(self): + return self._monthly_quota + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_ROUTING_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class TomTomRoutingConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['routing']['api_keys'] + monthly_quota = tomtom_server_conf['routing']['monthly_quota'] + tomtom_service_params = tomtom_server_conf['routing'].get('service', {}) + + routing_quota = self._get_quota() + soft_routing_limit = self._user_conf.get('soft_mapzen_routing_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('routing_log_path', None) + + return TomTomRoutingConfig(routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + routing_quota = self._org_conf.get('mapzen_routing_quota') or self._user_conf.get('mapzen_routing_quota') + if routing_quota is '': + return 0 + + return int(routing_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index 63f0b86..dec48c2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -1,3 +1,7 @@ +TOMTOM_ROUTING_APIKEY_ROUNDROBIN = 'tomtom_routing_apikey_roundrobin' +TOMTOM_GEOCODER_APIKEY_ROUNDROBIN = 'tomtom_geocoder_apikey_roundrobin' +TOMTOM_ISOLINES_APIKEY_ROUNDROBIN = 'tomtom_isolines_apikey_roundrobin' + PROFILE_DRIVING = 'car' PROFILE_CYCLING = 'bicycle' PROFILE_WALKING = 'pedestrian' diff --git a/server/lib/python/cartodb_services/test/credentials.py b/server/lib/python/cartodb_services/test/credentials.py index c818902..52a2a4d 100644 --- a/server/lib/python/cartodb_services/test/credentials.py +++ b/server/lib/python/cartodb_services/test/credentials.py @@ -1,9 +1,11 @@ import os + def mapbox_api_key(): """Returns Mapbox API key. Requires setting MAPBOX_API_KEY environment variable.""" return os.environ['MAPBOX_API_KEY'] + def tomtom_api_key(): """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable.""" return os.environ['TOMTOM_API_KEY'] diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 89f608e..868b963 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -7,7 +7,7 @@ from cartodb_services.metrics.config import * class TestGeocoderUserConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -28,6 +28,9 @@ class TestGeocoderUserConfig(TestCase): elif geocoder_provider == 'mapbox': assert geocoder_config.mapbox_geocoder is True assert geocoder_config.geocoding_quota == 100 + elif geocoder_provider == 'tomtom': + assert geocoder_config.tomtom_geocoder is True + assert geocoder_config.geocoding_quota == 100 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -81,7 +84,7 @@ class TestGeocoderUserConfig(TestCase): class TestGeocoderOrgConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -107,6 +110,9 @@ class TestGeocoderOrgConfig(TestCase): elif geocoder_provider == 'mapbox': assert geocoder_config.mapbox_geocoder is True assert geocoder_config.geocoding_quota == 200 + elif geocoder_provider == 'tomtom': + assert geocoder_config.tomtom_geocoder is True + assert geocoder_config.geocoding_quota == 200 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -167,7 +173,7 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() @@ -183,6 +189,8 @@ class TestIsolinesUserConfig(TestCase): assert isolines_config.service_type is 'mapzen_isolines' elif isolines_provider is 'mapbox': assert isolines_config.service_type is 'mapbox_isolines' + elif isolines_provider is 'tomtom': + assert isolines_config.service_type is 'tomtom_isolines' else: assert isolines_config.service_type is 'here_isolines' assert isolines_config.isolines_quota == 100 @@ -226,7 +234,7 @@ class TestIsolinesUserConfig(TestCase): class TestIsolinesOrgConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index a8761e1..d8bccea 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -80,6 +80,7 @@ def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}]) From c9d0f0447f1d7920eaab8699df0b718f613f41ae Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 16:40:12 +0100 Subject: [PATCH 016/265] Added server functions (untested) --- server/extension/sql/100_routing_helper.sql | 61 +++++++++ .../sql/105_route_between_points.sql | 8 ++ server/extension/sql/20_geocode_street.sql | 70 +++++++++- server/extension/sql/80_isolines_helper.sql | 124 ++++++++++++++++++ server/extension/sql/85_isodistance.sql | 17 +++ server/extension/sql/90_isochrone.sql | 16 +++ .../cartodb_services/tomtom/types.py | 6 + server/lib/python/cartodb_services/setup.py | 2 +- 8 files changed, 299 insertions(+), 5 deletions(-) diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index b57a3f9..b638f16 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -65,6 +65,67 @@ RETURNS cdb_dataservices_server.simple_route AS $$ service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( username TEXT, orgname TEXT, diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 25a149b..841ef39 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -29,6 +29,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$ mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -62,6 +66,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$ mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 9fd1f84..6466855 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -25,6 +25,9 @@ RETURNS Geometry AS $$ elif user_geocoder_config.mapbox_geocoder: mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: raise Exception('Requested geocoder is not available') @@ -82,8 +85,21 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -233,8 +249,54 @@ RETURNS Geometry AS $$ except BaseException as e: import sys service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using mapbox') + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') finally: service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 5e5ba38..24575d9 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -188,6 +188,70 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( username TEXT, orgname TEXT, @@ -311,3 +375,63 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ finally: service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index fc73199..18bd35f 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -24,6 +24,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -70,3 +73,17 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return result $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 1f3e798..8e5721f 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -24,6 +24,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -68,3 +71,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index dec48c2..4a7f082 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -18,3 +18,9 @@ MAX_SPEEDS = { PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed } + +TRANSPORT_MODE_TO_TOMTOM = { + 'car': 'car', + 'walk': 'pedestrian', + 'bicycle': 'bicycle', +} diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 901ee87..61c8a3c 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.16.7', + version='0.18.0', description='CartoDB Services API Python Library', From 74c0bb6f2689d2e81f9e884c20c670f0a99ed6c0 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 17:09:25 +0100 Subject: [PATCH 017/265] Created new server version (0.31.0) --- ...db_dataservices_server--0.30.2--0.31.0.sql | 449 ++ ...db_dataservices_server--0.31.0--0.30.2.sql | 175 + .../cdb_dataservices_server--0.31.0.sql | 3648 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.30.1--0.30.2.sql | 0 ...db_dataservices_server--0.30.2--0.30.1.sql | 0 .../cdb_dataservices_server--0.30.2.sql | 0 7 files changed, 4273 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.30.2--0.31.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.31.0--0.30.2.sql create mode 100644 server/extension/cdb_dataservices_server--0.31.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.1--0.30.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2--0.30.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.2--0.31.0.sql new file mode 100644 index 0000000..14e42d8 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.2--0.31.0.sql @@ -0,0 +1,449 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.31.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.30.2.sql b/server/extension/cdb_dataservices_server--0.31.0--0.30.2.sql new file mode 100644 index 0000000..bcad31d --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.31.0--0.30.2.sql @@ -0,0 +1,175 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(TEXT, TEXT, geometry(Point, 4326)[], TEXT); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isochrones(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql new file mode 100644 index 0000000..6da9f22 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -0,0 +1,3648 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index a140c0a..fe7e040 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.30.2' +default_version = '0.31.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2.sql From 12ac2c269d7366ab9ad899173e5cf1484fecab36 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 17:17:03 +0100 Subject: [PATCH 018/265] Modified expected result for test --- server/extension/test/expected/00_install_test.out | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index f87ca05..c8204d6 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -37,6 +37,12 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi (1 row) +SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); + cdb_conf_setconf +------------------ + +(1 row) + SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); cdb_conf_setconf ------------------ From da7d43cc08155e2f09644136ad5702672aac2020 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 17:38:02 +0100 Subject: [PATCH 019/265] Added client functions (untested) --- ...db_dataservices_client--0.23.0--0.24.0.sql | 202 + ...db_dataservices_client--0.24.0--0.23.0.sql | 25 + client/cdb_dataservices_client--0.24.0.sql | 4960 +++++++++++++++++ client/cdb_dataservices_client.control | 2 +- ...db_dataservices_client--0.22.0--0.23.0.sql | 0 ...db_dataservices_client--0.23.0--0.22.0.sql | 0 .../cdb_dataservices_client--0.23.0.sql | 0 client/renderer/interface.yaml | 28 + 8 files changed, 5216 insertions(+), 1 deletion(-) create mode 100644 client/cdb_dataservices_client--0.23.0--0.24.0.sql create mode 100644 client/cdb_dataservices_client--0.24.0--0.23.0.sql create mode 100644 client/cdb_dataservices_client--0.24.0.sql rename client/{ => old_versions}/cdb_dataservices_client--0.22.0--0.23.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.23.0--0.22.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.23.0.sql (100%) diff --git a/client/cdb_dataservices_client--0.23.0--0.24.0.sql b/client/cdb_dataservices_client--0.23.0--0.24.0.sql new file mode 100644 index 0000000..c611977 --- /dev/null +++ b/client/cdb_dataservices_client--0.23.0--0.24.0.sql @@ -0,0 +1,202 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.24.0'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; diff --git a/client/cdb_dataservices_client--0.24.0--0.23.0.sql b/client/cdb_dataservices_client--0.24.0--0.23.0.sql new file mode 100644 index 0000000..1f5abc6 --- /dev/null +++ b/client/cdb_dataservices_client--0.24.0--0.23.0.sql @@ -0,0 +1,25 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.23.0'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_geocode_street_point (text, text, text, text); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_isochrone (geometry(Geometry, 4326), text, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_isodistance (geometry(Geometry, 4326), text, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (text, text, text, text); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (geometry(Geometry, 4326), text, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (geometry(Geometry, 4326), text, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); diff --git a/client/cdb_dataservices_client--0.24.0.sql b/client/cdb_dataservices_client--0.24.0.sql new file mode 100644 index 0000000..14a3af6 --- /dev/null +++ b/client/cdb_dataservices_client--0.24.0.sql @@ -0,0 +1,4960 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- For the OBS_Meta functions +CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params) + ) + ) + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + # Create a new table with the required columns + plpy.execute('CREATE TABLE "{schema}".{table_name} ( ' + 'cartodb_id int, the_geom geometry, {columns_with_types} ' + ');' + .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute( + "SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params))) + + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [ + colnames_arr[i] + + ' ' + + coltypes_arr[i] for i in range( + 0, + len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + aliased_colname_list = ','.join( + ['result.' + name for name in colnames_arr]) + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute( + "SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, " + "{schema}::text, {dbname}::text, {table_name}::text);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + user_db_role=plpy.quote_literal(user_db_role), + schema=plpy.quote_literal(user_schema), + dbname=plpy.quote_literal(dbname), + table_name=plpy.quote_literal(table_name))) + + if ds_fdw_metadata[0]["schemaname"]: + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + else: + raise Exception('Error connecting dataset via FDW') + + # Create a new table with the required columns + plpy.execute( + 'INSERT INTO "{schema}".{analysis_table_name} ' + 'SELECT ut.cartodb_id, ut.the_geom, {colname_list} ' + 'FROM "{schema}".{table_name} ut ' + 'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, ' + '{function_name}::text, {params}::json) ' + 'AS result ({columns_with_types}, cartodb_id int) ' + 'ON result.cartodb_id = ut.cartodb_id;' .format( + schema=user_schema, + analysis_table_name=output_table_name, + colname_list=aliased_colname_list, + table_name=table_name, + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params), + columns_with_types=columns_with_types)) + + # Wipe user FDW data from the server + wiped = plpy.execute( + "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, " + "{server_table_name}::text, {fdw_server}::text)" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + fdw_server=plpy.quote_literal(server_name))) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text +)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata( + username text, + orgname text, + function_name text, + params json +) RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData( + username text, + orgname text, + table_schema text, + table_name text, + function_name text, + params json +) RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( + username text, + orgname text, + table_schema text, + table_name text, + server_name text +) RETURNS boolean AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, orgname, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, orgname, ip_address); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, orgname, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_with_waypoints (username, orgname, waypoints, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundary (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundaryid (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundarybyid (username, orgname, geometry_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasure (username, orgname, geom, measure_id, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasurebyid (username, orgname, geom_ref, measure_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomvals, params, merge); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json) +RETURNS TABLE(id text, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomrefs, params); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeta (username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_metadatavalidation (username, orgname, geom_extent, geom_type, params, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getcategory (username, orgname, geom, category_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, orgname, geom, name, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensuscategory (username, orgname, geom, name, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getpopulation (username, orgname, geom, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_search (username, orgname, search_term, relevant_boundary); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailableboundaries (username, orgname, geom, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_dumpversion (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablenumerators (username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, section_tags text[] DEFAULT ARRAY[]::TEXT[], subsection_tags text[] DEFAULT ARRAY[]::TEXT[], other_tags text[] DEFAULT ARRAY[]::TEXT[], ids text[] DEFAULT ARRAY[]::TEXT[], name text DEFAULT NULL, denom_id text DEFAULT '', geom_id text DEFAULT '', timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._obs_getnumerators (username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabledenominators (username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, timespan text DEFAULT NULL, number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablegeometries (username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabletimespans (username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_legacybuildermetadata (username, orgname, aggregate_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info (username text, orgname text) +RETURNS SETOF service_quota_info AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) +RETURNS BOOLEAN AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_enough_quota (username, orgname, service, input_size); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck( + source_query text, + parameters json +) RETURNS boolean AS $$ +DECLARE + errors text[]; + validator_errors text[]; + validator_error text; + valid boolean; + geoms record; +BEGIN + errors := (ARRAY[])::TEXT[]; + FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type, + bool_and(st_isvalid(the_geom)) as valid, + avg(st_npoints(the_geom)) as avg_vertex, + ST_SetSRID(ST_Extent(the_geom), 4326) as extent, + count(*)::INT as numgeoms + FROM (%s) as _source + GROUP BY ST_GeometryType(the_geom)', source_query) + LOOP + IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN + errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type)); + END IF; + + IF geoms.valid IS FALSE THEN + errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them'); + END IF; + + -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here + -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813 + IF geoms.avg_vertex > 1000 THEN + errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them'); + END IF; + + -- OBS specific part + EXECUTE 'SELECT valid, errors + FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)' + INTO valid, validator_errors + USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms; + IF valid is FALSE THEN + FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors + LOOP + errors := array_append(errors, validator_error); + END LOOP; + END IF; + END LOOP; + + IF CARDINALITY(errors) > 0 THEN + RAISE EXCEPTION '%', errors; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_with_waypoints(waypoints geometry(Point, 4326)[], mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom Geometry, category_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom Geometry, name text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom Geometry, name text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text ) TO publicuser; + + + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index ca33b14..b56df21 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.23.0' +default_version = '0.24.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/cdb_dataservices_client--0.22.0--0.23.0.sql b/client/old_versions/cdb_dataservices_client--0.22.0--0.23.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.22.0--0.23.0.sql rename to client/old_versions/cdb_dataservices_client--0.22.0--0.23.0.sql diff --git a/client/cdb_dataservices_client--0.23.0--0.22.0.sql b/client/old_versions/cdb_dataservices_client--0.23.0--0.22.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.23.0--0.22.0.sql rename to client/old_versions/cdb_dataservices_client--0.23.0--0.22.0.sql diff --git a/client/cdb_dataservices_client--0.23.0.sql b/client/old_versions/cdb_dataservices_client--0.23.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.23.0.sql rename to client/old_versions/cdb_dataservices_client--0.23.0.sql diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 5b321d8..874e99a 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -94,6 +94,14 @@ - { name: state_province, type: text, default: 'NULL'} - { name: country, type: text, default: 'NULL'} +- name: cdb_tomtom_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + - name: cdb_mapzen_geocode_street_point return_type: Geometry params: @@ -132,6 +140,16 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_tomtom_isochrone + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_mapzen_isochrone return_type: SETOF cdb_dataservices_client.isoline multi_row: true @@ -152,6 +170,16 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_tomtom_isodistance + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_mapzen_isodistance return_type: SETOF cdb_dataservices_client.isoline multi_row: true From ffa1080277762e06a0e2e2550f23c743d871dc65 Mon Sep 17 00:00:00 2001 From: Antonio Date: Tue, 13 Mar 2018 14:58:40 +0100 Subject: [PATCH 020/265] isochrones fix --- .../cartodb_services/cartodb_services/tomtom/isolines.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py index 83bbfe1..a6fc062 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py @@ -54,7 +54,7 @@ class TomTomIsolines(): [x for x in VALID_PROFILES]))) def _parse_coordinates(self, boundary): - return [Coordinate(c[ENTRY_LONGITUDE], c[ENTRY_LONGITUDE]) for c in boundary] + return [Coordinate(c[ENTRY_LONGITUDE], c[ENTRY_LATITUDE]) for c in boundary] def _parse_reachablerange_response(self, response): json_response = json.loads(response) @@ -107,7 +107,6 @@ class TomTomIsolines(): time_range=time_range, profile=profile, date_time=date_time) - isochrones.append(TomTomIsochronesResponse(coordinates, time_range)) return isochrones From 59751d7c9cb1fc1bb98dd3154db03e50df285dbb Mon Sep 17 00:00:00 2001 From: Antonio Date: Tue, 13 Mar 2018 16:01:05 +0100 Subject: [PATCH 021/265] Avoid reaching provider for empty geocodings --- server/extension/sql/20_geocode_street.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 9fd1f84..e4af4b9 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -98,6 +98,11 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: @@ -130,6 +135,11 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits(quota=False) + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: @@ -166,6 +176,11 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) country_iso3 = None if country: @@ -208,6 +223,11 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) country_iso3166 = None From 1c38b33501cf4b6f7d2f7449e61785c8bb001c8b Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 14 Mar 2018 09:55:07 +0100 Subject: [PATCH 022/265] Updated server side scripts --- NEWS.md | 5 + ...db_dataservices_server--0.30.2--0.30.3.sql | 177 + ...db_dataservices_server--0.30.3--0.30.2.sql | 160 + .../cdb_dataservices_server--0.30.3.sql | 3380 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.30.1--0.30.2.sql | 0 ...db_dataservices_server--0.30.2--0.30.1.sql | 0 .../cdb_dataservices_server--0.30.2.sql | 0 8 files changed, 3723 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.3.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.1--0.30.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2--0.30.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2.sql (100%) diff --git a/NEWS.md b/NEWS.md index 53c6bab..72fb009 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,9 @@ +March 14th, 2018 +================== +* Version `0.30.3` of server side + * Avoid reaching provider for empty geocodings. Stopping in the server side PL/Python functions + February 22th, 2018 ================== * Version `0.17.2` of the python library diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql b/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql new file mode 100644 index 0000000..165b287 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql @@ -0,0 +1,177 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql b/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql new file mode 100644 index 0000000..f33b24c --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql @@ -0,0 +1,160 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.3.sql b/server/extension/cdb_dataservices_server--0.30.3.sql new file mode 100644 index 0000000..a903b66 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.3.sql @@ -0,0 +1,3380 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + + if not searchtext and not city and not state_province and not country: + service_manager.quota_service.increment_empty_service_use() + return None + + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index a140c0a..7b0691b 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.30.2' +default_version = '0.30.3' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2.sql From f652a52a8dadaa892f8ceed7541442d31dba3238 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 14 Mar 2018 11:19:48 +0100 Subject: [PATCH 023/265] Use routing quota instead of monthly quota --- .../cartodb_services/cartodb_services/metrics/config.py | 9 +++------ .../cartodb_services/cartodb_services/metrics/quota.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 6179d1b..9344af3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -153,7 +153,7 @@ class RoutingConfig(ServiceConfig): elif self._routing_provider == self.MAPBOX_PROVIDER: self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys self._mapbox_service_params = self._db_config.mapbox_routing_service_params - self._set_monthly_quota() + self._routing_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -193,8 +193,8 @@ class RoutingConfig(ServiceConfig): return self._mapbox_service_params @property - def monthly_quota(self): - return self._monthly_quota + def routing_quota(self): + return self._routing_quota @property def period_end_date(self): @@ -204,9 +204,6 @@ class RoutingConfig(ServiceConfig): def soft_limit(self): return self._soft_limit - def _set_monthly_quota(self): - self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) - def _set_soft_limit(self): if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_limit = True diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 7be4c6c..b28462e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -122,7 +122,7 @@ class QuotaChecker: return False def __check_routing_quota(self): - user_quota = self._user_service_config.monthly_quota + user_quota = self._user_service_config.routing_quota today = date.today() service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) From 0196292093cd793bc07cec58ad53041a590fbce6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 14 Mar 2018 11:20:12 +0100 Subject: [PATCH 024/265] Bump version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index c1c5893..a3c0682 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.2', + version='0.17.3', description='CartoDB Services API Python Library', From e324afd77f3ef13a111972d881282e9d74a1fce9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 14 Mar 2018 11:20:56 +0100 Subject: [PATCH 025/265] Update NEWS.md --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 53c6bab..389ed78 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,9 @@ +March 14th, 2018 +================ +* Version `0.17.3` of the python library + * Fix bug with Mapbox routing not using the proper quota value + February 22th, 2018 ================== * Version `0.17.2` of the python library From f5cbc195cc2619570f4bf08896093947e7c45b9d Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 14 Mar 2018 13:31:00 +0100 Subject: [PATCH 026/265] Fixed tests --- .../python/cartodb_services/test/metrics/test_config.py | 4 ++-- .../python/cartodb_services/test/metrics/test_quota.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 89f608e..d233f4b 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -301,7 +301,7 @@ class TestRoutingConfig(TestCase): self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) orgname = None config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) - assert config.monthly_quota == 1000 + assert config.routing_quota == 1000 def test_org_quota_overrides_user_quota(self): self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) @@ -315,7 +315,7 @@ class TestRoutingConfig(TestCase): self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0) config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) - assert config.monthly_quota == 5000 + assert config.routing_quota == 5000 def test_should_have_soft_limit_false_by_default(self): orgname = None diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index 079c31d..843d90a 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -30,7 +30,7 @@ class TestQuotaChecker(TestCase): username = self.username, organization = None, service_type = self.service_type, - monthly_quota = 1000, + routing_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) @@ -43,7 +43,7 @@ class TestQuotaChecker(TestCase): username = self.username, organization = None, service_type = self.service_type, - monthly_quota = 1000, + routing_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) @@ -61,7 +61,7 @@ class TestQuotaChecker(TestCase): username = self.username, organization = None, service_type = self.service_type, - monthly_quota = 1000, + routing_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) @@ -75,7 +75,7 @@ class TestQuotaChecker(TestCase): username = self.username, organization = None, service_type = self.service_type, - monthly_quota = 1000, + routing_quota = 1000, period_end_date = datetime.today(), soft_limit = True ) From c90859e58b529442d8862971bf8e2d3d4b1dfbcc Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 14 Mar 2018 16:26:00 +0100 Subject: [PATCH 027/265] Fixed error when checking quota --- NEWS.md | 2 ++ .../cartodb_services/cartodb_services/metrics/config.py | 4 ++++ server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 389ed78..6ea9601 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,8 @@ March 14th, 2018 ================ +* Version `0.17.4` of the python library + * Fix bug with previous version when checking quotas * Version `0.17.3` of the python library * Fix bug with Mapbox routing not using the proper quota value diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 9344af3..07f742c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -196,6 +196,10 @@ class RoutingConfig(ServiceConfig): def routing_quota(self): return self._routing_quota + @property + def monthly_quota(self): + return self._routing_quota + @property def period_end_date(self): return self._period_end_date diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index a3c0682..844be14 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.3', + version='0.17.4', description='CartoDB Services API Python Library', From 71c93fe13adcfbb446a5436dd9902e5a16a96620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Mart=C3=ADn?= Date: Thu, 15 Mar 2018 15:58:15 +0100 Subject: [PATCH 028/265] Add ST_MakeValid to sql --- server/extension/sql/80_isolines_helper.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 5e5ba38..800c362 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ # -- TODO encapsulate this block into a func/method locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates) multipolygon = plpy.execute(sql, 1)[0]['geom'] else: multipolygon = None From 44fae489a38a07effa75177533e777cb8256aa38 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 15 Mar 2018 15:09:21 +0000 Subject: [PATCH 029/265] Update scripts --- ...db_dataservices_server--0.30.2--0.30.3.sql | 70 + ...db_dataservices_server--0.30.3--0.30.2.sql | 70 + .../cdb_dataservices_server--0.30.3.sql | 3360 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.30.1--0.30.2.sql | 0 ...db_dataservices_server--0.30.2--0.30.1.sql | 0 .../cdb_dataservices_server--0.30.2.sql | 0 7 files changed, 3501 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.3.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.1--0.30.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2--0.30.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql b/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql new file mode 100644 index 0000000..a07a840 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql @@ -0,0 +1,70 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql b/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql new file mode 100644 index 0000000..d950acb --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql @@ -0,0 +1,70 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.3.sql b/server/extension/cdb_dataservices_server--0.30.3.sql new file mode 100644 index 0000000..62baa15 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.3.sql @@ -0,0 +1,3360 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index a140c0a..7b0691b 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.30.2' +default_version = '0.30.3' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.1--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.1--0.30.2.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2--0.30.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.1.sql diff --git a/server/extension/cdb_dataservices_server--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2.sql From 5d20d0bdf507382ad33a6cde0dd0e89af114edb7 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 11:16:57 +0100 Subject: [PATCH 030/265] Updated NEWS.md --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 6ea9601..83d3fa9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,9 @@ +March 16th, 2018 +================ +* Version `0.30.3` of server side + * Fix problem with invalid Mapbox isolines + March 14th, 2018 ================ * Version `0.17.4` of the python library From e15c6127d372f545fedd21cfdfd05ebab798e8a3 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 12:01:12 +0100 Subject: [PATCH 031/265] Added ST_CollectionExtract --- server/extension/sql/80_isolines_helper.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 800c362..e8043fe 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ # -- TODO encapsulate this block into a func/method locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) multipolygon = plpy.execute(sql, 1)[0]['geom'] else: multipolygon = None From 6017b53ea0ba86909776b60df32a60c6674db3d0 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 12:07:20 +0100 Subject: [PATCH 032/265] Server version 0.30.4 --- NEWS.md | 2 + ...db_dataservices_server--0.30.3--0.30.4.sql | 69 + ...db_dataservices_server--0.30.4--0.30.3.sql | 69 + .../cdb_dataservices_server--0.30.4.sql | 3360 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.30.2--0.30.3.sql | 0 ...db_dataservices_server--0.30.3--0.30.2.sql | 0 .../cdb_dataservices_server--0.30.3.sql | 0 8 files changed, 3501 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.4.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.2--0.30.3.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.3--0.30.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.3.sql (100%) diff --git a/NEWS.md b/NEWS.md index 83d3fa9..e30530c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,8 @@ March 16th, 2018 ================ +* Version `0.30.4` of server side + * Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries * Version `0.30.3` of server side * Fix problem with invalid Mapbox isolines diff --git a/server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql b/server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql new file mode 100644 index 0000000..5db9881 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql @@ -0,0 +1,69 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql b/server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql new file mode 100644 index 0000000..b875bdb --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql @@ -0,0 +1,69 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.4.sql b/server/extension/cdb_dataservices_server--0.30.4.sql new file mode 100644 index 0000000..db7b1ba --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.4.sql @@ -0,0 +1,3360 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 7b0691b..51ab581 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.30.3' +default_version = '0.30.4' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.3.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.3.sql diff --git a/server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.2.sql diff --git a/server/extension/cdb_dataservices_server--0.30.3.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.3.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.3.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.3.sql From ace7683ffe45107821134137842f1129eec961f6 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 12:14:14 +0100 Subject: [PATCH 033/265] Added suggestion from CR --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index e30530c..631221e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ March 16th, 2018 ================ * Version `0.30.4` of server side * Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries + +March 16th, 2018 +================ * Version `0.30.3` of server side * Fix problem with invalid Mapbox isolines From d66804b93dcb18d3828aa1132a0856a25234bb7e Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 14:47:56 +0100 Subject: [PATCH 034/265] Fixed test --- .../lib/python/cartodb_services/test/test_tomtomgeocoder.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py index 706b7de..7fcfcda 100644 --- a/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py @@ -6,8 +6,6 @@ from credentials import tomtom_api_key INVALID_APIKEY = 'invalid_apikey' VALID_ADDRESS = 'Plaza Mayor 3, Valladolid' -WELL_KNOWN_LONGITUDE = -4.728 -WELL_KNOWN_LATITUDE = 41.653 class TomTomGeocoderTestCase(unittest.TestCase): @@ -22,8 +20,7 @@ class TomTomGeocoderTestCase(unittest.TestCase): def test_valid_request(self): place = self.geocoder.geocode(VALID_ADDRESS) - self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE) - self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE) + assert place def test_valid_request_namedplace(self): place = self.geocoder.geocode(searchtext='Barcelona') From e78063ae75c0b5c18408eaa96782b7d1ff2f75e8 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 16:01:00 +0100 Subject: [PATCH 035/265] Fixed geocoding validity --- .../cartodb_services/tomtom/geocoder.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 2bd0525..783be25 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -48,18 +48,31 @@ class TomTomGeocoder(Traceable): latitude = position[ENTRY_LAT] return [longitude, latitude] + def _validate_input(self, searchtext, city=None, state_province=None, + country=None): + if searchtext and searchtext.strip(): + return True + elif city: + return True + elif state_province: + return True + + return False + @qps_retry(qps=5) def geocode(self, searchtext, city=None, state_province=None, country=None): - if searchtext and searchtext.strip(): - address = [normalize(searchtext)] - if city: - address.append(normalize(city)) - if state_province: - address.append(normalize(state_province)) - else: + if not self._validate_input(searchtext, city, state_province, country): return [] + address = [] + if searchtext and searchtext.strip(): + address.append(normalize(searchtext)) + if city: + address.append(normalize(city)) + if state_province: + address.append(normalize(state_province)) + uri = self._uri(searchtext=', '.join(address), countries=country) try: From a34da0adb589b46e1c11e678efdf6bcf80eed734 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 16:37:17 +0100 Subject: [PATCH 036/265] Fixed CR suggestions --- server/extension/sql/20_geocode_street.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 0e6ca19..9fd1f84 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -98,7 +98,6 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() - geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: @@ -131,7 +130,6 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits(quota=False) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: @@ -168,7 +166,6 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() - geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) country_iso3 = None if country: @@ -211,7 +208,6 @@ RETURNS Geometry AS $$ try: service_manager.assert_within_limits() - geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) country_iso3166 = None From 552a7d4886f7540a009b8adcaf76cc45aaf8f606 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 16 Mar 2018 16:55:08 +0100 Subject: [PATCH 037/265] Fixing Mapbox geocoding validity checks --- NEWS.md | 2 +- .../cartodb_services/mapbox/geocoder.py | 27 ++++++++++++++----- server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1b975bb..631221e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -683,4 +683,4 @@ Nov 27, 2015: * Release Geocoder API BETA 1 * Added the organization public user to the api key check -https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta \ No newline at end of file +https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index d6acb98..a8a9de0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -64,18 +64,31 @@ class MapboxGeocoder(Traceable): latitude = location[1] return [longitude, latitude] + def _validate_input(self, searchtext, city=None, state_province=None, + country=None): + if searchtext and searchtext.strip(): + return True + elif city: + return True + elif state_province: + return True + + return False + @qps_retry(qps=10) def geocode(self, searchtext, city=None, state_province=None, country=None): - if searchtext and searchtext.strip(): - address = [normalize(searchtext)] - if city: - address.append(normalize(city)) - if state_province: - address.append(normalize(state_province)) - else: + if not self._validate_input(searchtext, city, state_province, country): return [] + address = [] + if searchtext and searchtext.strip(): + address.append(normalize(searchtext)) + if city: + address.append(normalize(city)) + if state_province: + address.append(normalize(state_province)) + country = [country] if country else None try: diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 844be14..7cc70f8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.4', + version='0.17.5', description='CartoDB Services API Python Library', From ea5818f08f1869a8a326a85f3dac72a61aded99b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Mar 2018 16:20:29 +0200 Subject: [PATCH 038/265] Update NEWS.md --- NEWS.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index 631221e..3837af8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,25 +1,26 @@ - -March 16th, 2018 +March 27th, 2018 ================ -* Version `0.30.4` of server side - * Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries +* Version `0.17.5` of python library + * Avoid reaching provider for empty geocodings (but still incrementing empty service use) #476 March 16th, 2018 ================ * Version `0.30.3` of server side - * Fix problem with invalid Mapbox isolines + * Fix problem with invalid Mapbox isolines #483 +* Version `0.30.4` of server side + * Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries #486 March 14th, 2018 ================ * Version `0.17.4` of the python library - * Fix bug with previous version when checking quotas + * Fix bug with previous version when checking quotas #480 * Version `0.17.3` of the python library - * Fix bug with Mapbox routing not using the proper quota value + * Fix bug with Mapbox routing not using the proper quota value #477 February 22th, 2018 ================== * Version `0.17.2` of the python library - * Fix bug with Mapbox isolines not stopping at the seacoast + * Fix bug with Mapbox isolines not stopping at the seacoast #471 February 27th, 2018 ================== From 2a47000f320e089afe19dcca97729a8cd97dc34b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 23 Mar 2018 12:46:51 +0100 Subject: [PATCH 039/265] coordinates_to_polygon should always return multipolygon to avoid inconsistencies --- server/extension/sql/80_isolines_helper.sql | 2 +- .../python/cartodb_services/cartodb_services/mapzen/types.py | 2 +- .../cartodb_services/cartodb_services/tools/coordinates.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index e8043fe..953be10 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ # -- TODO encapsulate this block into a func/method locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) multipolygon = plpy.execute(sql, 1)[0]['geom'] else: multipolygon = None diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py index 986eb91..f5ad462 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py @@ -28,7 +28,7 @@ def coordinates_to_polygon(coordinates): wkt_coordinates = ','.join(result_coordinates) try: - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3)) as geom".format(wkt_coordinates) geometry = plpy.execute(sql, 1)[0]['geom'] except BaseException as e: plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e)) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py b/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py index b02e801..6688970 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py @@ -55,7 +55,7 @@ def coordinates_to_polygon(coordinates): wkt_coordinates = ','.join(result_coordinates) try: - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3)) as geom".format(wkt_coordinates) geometry = plpy.execute(sql, 1)[0]['geom'] except BaseException as e: plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e)) From 8025f657b81cf245915ba691dee607c7330ac12c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 23 Mar 2018 14:21:43 +0100 Subject: [PATCH 040/265] Upgrade version scripts --- ...db_dataservices_server--0.30.4--0.30.5.sql | 68 + ...db_dataservices_server--0.30.5--0.30.4.sql | 68 + .../cdb_dataservices_server--0.30.5.sql | 3360 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.30.3--0.30.4.sql | 0 ...db_dataservices_server--0.30.4--0.30.3.sql | 0 .../cdb_dataservices_server--0.30.4.sql | 0 7 files changed, 3497 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql create mode 100644 server/extension/cdb_dataservices_server--0.30.5.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.3--0.30.4.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.4--0.30.3.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.4.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql b/server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql new file mode 100644 index 0000000..7f5648a --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql @@ -0,0 +1,68 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.5'" to load this file. \quit + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql b/server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql new file mode 100644 index 0000000..eaff320 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql @@ -0,0 +1,68 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.30.5.sql b/server/extension/cdb_dataservices_server--0.30.5.sql new file mode 100644 index 0000000..ca08190 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.5.sql @@ -0,0 +1,3360 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 51ab581..ff4cd53 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.30.4' +default_version = '0.30.5' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.4.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.3--0.30.4.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.4.sql diff --git a/server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.3.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.4--0.30.3.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.3.sql diff --git a/server/extension/cdb_dataservices_server--0.30.4.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.4.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.4.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.4.sql From e25e2c26a04fedf48204d4f518276dfed06a8972 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Mar 2018 12:57:32 +0200 Subject: [PATCH 041/265] Remove the PARALLEL modifiers from the version SQL artifact --- .../cdb_dataservices_server--0.30.5.sql | 256 +++++++++--------- 1 file changed, 128 insertions(+), 128 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.30.5.sql b/server/extension/cdb_dataservices_server--0.30.5.sql index ca08190..f8c24a5 100644 --- a/server/extension/cdb_dataservices_server--0.30.5.sql +++ b/server/extension/cdb_dataservices_server--0.30.5.sql @@ -66,7 +66,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate Mapbox routing') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( username TEXT, @@ -127,7 +127,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('Error trying to calculate mapzen routing') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( username TEXT, orgname TEXT, @@ -161,7 +161,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( @@ -194,7 +194,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- Get the connection to redis from cache or create a new one CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) RETURNS boolean AS $$ @@ -212,7 +212,7 @@ RETURNS boolean AS $$ 'redis_metrics_connection': redis_metrics_connection, } return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; -- -- Observatory connection config -- @@ -229,7 +229,7 @@ RETURNS text AS $$ user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] return user_obs_config.connection_str -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( username TEXT, @@ -240,7 +240,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJS RETURNS json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( username TEXT, @@ -283,7 +283,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( username TEXT, @@ -294,7 +294,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( RETURNS SETOF json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( username TEXT, @@ -340,7 +340,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( username TEXT, @@ -350,7 +350,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( RETURNS json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( username TEXT, @@ -392,7 +392,7 @@ RETURNS json AS $$ raise Exception('Error trying to obs_get_segment_snapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( username TEXT, @@ -402,7 +402,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( RETURNS SETOF json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( username TEXT, @@ -447,7 +447,7 @@ RETURNS SETOF JSON AS $$ raise Exception('Error trying to OBS_GetSegmentSnapshot') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( username TEXT, @@ -460,7 +460,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( RETURNS NUMERIC AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( username TEXT, @@ -504,7 +504,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( username TEXT, @@ -516,7 +516,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( RETURNS TEXT AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( username TEXT, @@ -559,7 +559,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( username TEXT, @@ -572,7 +572,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( RETURNS NUMERIC AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( username TEXT, @@ -616,7 +616,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetUSCensusMeasure') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( username TEXT, @@ -628,7 +628,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( RETURNS TEXT AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( username TEXT, @@ -671,7 +671,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetUSCensusCategory') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( username TEXT, @@ -683,7 +683,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( RETURNS NUMERIC AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( username TEXT, @@ -726,7 +726,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetPopulation') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( username TEXT, @@ -738,7 +738,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( RETURNS NUMERIC AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( username TEXT, @@ -781,7 +781,7 @@ RETURNS NUMERIC AS $$ raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -795,7 +795,7 @@ RETURNS TABLE ( ) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( username TEXT, @@ -842,7 +842,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomvals)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( username TEXT, @@ -855,7 +855,7 @@ RETURNS TABLE ( ) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( username TEXT, @@ -903,7 +903,7 @@ RETURNS TABLE ( raise Exception('Error trying to OBS_GetData') finally: quota_service.increment_total_service_use(len(geomrefs)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( username TEXT, @@ -916,7 +916,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( RETURNS JSON AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( username TEXT, @@ -951,7 +951,7 @@ RETURNS JSON AS $$ import sys logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_GetMeta') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( username TEXT, @@ -963,7 +963,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( RETURNS TABLE(valid boolean, errors text[]) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( username TEXT, @@ -997,7 +997,7 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$ import sys logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to OBS_MetadataValidation') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( username TEXT, orgname TEXT, @@ -1006,7 +1006,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( username TEXT, @@ -1055,7 +1055,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) raise Exception('Error trying to OBS_Search') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( username TEXT, @@ -1065,7 +1065,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( username TEXT, @@ -1113,7 +1113,7 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( username TEXT, orgname TEXT, @@ -1123,7 +1123,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( RETURNS geometry(Geometry, 4326) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( username TEXT, @@ -1165,7 +1165,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundary') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( username TEXT, @@ -1176,7 +1176,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( RETURNS TEXT AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( username TEXT, @@ -1218,7 +1218,7 @@ RETURNS TEXT AS $$ raise Exception('Error trying to OBS_GetBoundaryId') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( username TEXT, @@ -1229,7 +1229,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( RETURNS geometry(Geometry, 4326) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( username TEXT, @@ -1271,7 +1271,7 @@ RETURNS geometry(Geometry, 4326) AS $$ raise Exception('Error trying to OBS_GetBoundaryById') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( username TEXT, @@ -1283,7 +1283,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( username TEXT, @@ -1331,7 +1331,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( username TEXT, @@ -1344,7 +1344,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndR RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( username TEXT, @@ -1393,7 +1393,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( username TEXT, @@ -1405,7 +1405,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( username TEXT, @@ -1453,7 +1453,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByGeometry') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( username TEXT, @@ -1466,7 +1466,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadiu RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( username TEXT, @@ -1515,7 +1515,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ raise Exception('Error trying to OBS_GetPointsByPointAndRadius') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); @@ -1526,37 +1526,37 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) )[0] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); TARGET cdb_observatory._OBS_ConnectUserTable; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) RETURNS cdb_dataservices_server.ds_return_metadata AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); TARGET cdb_observatory._OBS_GetReturnMetadata; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) RETURNS SETOF record AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); TARGET cdb_observatory._OBS_FetchJoinFdwTableData; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) RETURNS boolean AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); TARGET cdb_observatory._OBS_DisconnectUserTable; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) RETURNS text AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT cdb_observatory.obs_dumpversion(); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; -- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); @@ -1572,7 +1572,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( username TEXT, @@ -1589,7 +1589,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); @@ -1604,7 +1604,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); @@ -1620,7 +1620,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); @@ -1635,7 +1635,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( username TEXT, @@ -1644,7 +1644,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( RETURNS TABLE(name TEXT, subsection JSON) AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plproxy VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() RETURNS boolean AS $$ cache_key = "logger_config" @@ -1655,13 +1655,13 @@ RETURNS boolean AS $$ logger_config = LoggerConfig(plpy) GD[cache_key] = logger_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; -- This is done in order to avoid an undesired depedency on cartodb extension CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) RETURNS JSON AS $$ SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; -$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; +$$ LANGUAGE SQL SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) @@ -1670,7 +1670,7 @@ BEGIN PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; END -$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) @@ -1678,7 +1678,7 @@ FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) BEGIN EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; END -$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) @@ -1693,7 +1693,7 @@ RETURNS boolean AS $$ geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ @@ -1707,7 +1707,7 @@ RETURNS boolean AS $$ geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) GD[cache_key] = geocoder_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -1721,7 +1721,7 @@ RETURNS boolean AS $$ isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = isolines_routing_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) RETURNS boolean AS $$ @@ -1749,7 +1749,7 @@ RETURNS boolean AS $$ obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) GD[cache_key] = obs_snapshot_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) RETURNS boolean AS $$ @@ -1763,7 +1763,7 @@ RETURNS boolean AS $$ obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) GD[cache_key] = obs_config return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) @@ -1855,7 +1855,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( @@ -1872,7 +1872,7 @@ returns BOOLEAN AS $$ WHERE p.service = service_::cdb_dataservices_server.service_type; RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); END -$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpgsql STABLE ; -- Geocodes a street address given a searchtext and a state and/or country CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -1903,7 +1903,7 @@ RETURNS Geometry AS $$ else: raise Exception('Requested geocoder is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -1919,7 +1919,7 @@ RETURNS Geometry AS $$ else: raise Exception('Here geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -1934,7 +1934,7 @@ RETURNS Geometry AS $$ else: raise Exception('Google geocoder is not available for your account.') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -1947,7 +1947,7 @@ RETURNS Geometry AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -1960,7 +1960,7 @@ RETURNS Geometry AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -1993,7 +1993,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using here maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -2025,7 +2025,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using google maps') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -2066,7 +2066,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using mapzen') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ @@ -2112,7 +2112,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode street point using mapbox') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( username TEXT, @@ -2131,7 +2131,7 @@ RETURNS JSON AS $$ return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) else: return None -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( username TEXT, @@ -2155,7 +2155,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_user_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpythonu VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( username TEXT, @@ -2179,7 +2179,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_org_rate_limits(config) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( username TEXT, @@ -2203,7 +2203,7 @@ RETURNS VOID AS $$ period = None config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpythonu VOLATILE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService @@ -2238,7 +2238,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin0 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -------------------------------------------------------------------------------- @@ -2258,7 +2258,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; ---- cdb_geocode_admin1_polygon(admin1_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ @@ -2295,7 +2295,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) @@ -2333,7 +2333,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -------------------------------------------------------------------------------- @@ -2362,7 +2362,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) @@ -2386,7 +2386,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; ---- cdb_geocode_namedplace_point(city_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) @@ -2406,7 +2406,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; ---- cdb_geocode_namedplace_point(city_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) @@ -2426,7 +2426,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) @@ -2446,7 +2446,7 @@ RETURNS Geometry AS $$ logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ @@ -2489,7 +2489,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -------------------------------------------------------------------------------- @@ -2513,7 +2513,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; ---- cdb_geocode_namedplace_point(city_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) @@ -2533,7 +2533,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) @@ -2557,7 +2557,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) RETURNS Geometry AS $$ @@ -2593,12 +2593,12 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; +$$ LANGUAGE SQL STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) @@ -2635,12 +2635,12 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; +$$ LANGUAGE SQL STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) @@ -2677,12 +2677,12 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; +$$ LANGUAGE SQL STABLE ; @@ -2720,12 +2720,12 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; +$$ LANGUAGE SQL STABLE ; -------------------------------------------------------------------------------- @@ -2750,7 +2750,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) RETURNS Geometry AS $$ @@ -2775,7 +2775,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) RETURNS Geometry AS $$ @@ -2796,7 +2796,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) RETURNS Geometry AS $$ @@ -2821,7 +2821,7 @@ RETURNS Geometry AS $$ RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics @@ -2856,7 +2856,7 @@ RETURNS Geometry AS $$ raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -------------------------------------------------------------------------------- @@ -2887,7 +2887,7 @@ RETURNS Geometry AS $$ FROM matches; RETURN ret; END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +$$ LANGUAGE plpgsql STABLE ; CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) @@ -2945,7 +2945,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( username TEXT, @@ -3011,7 +3011,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( username TEXT, @@ -3076,7 +3076,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isolines') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( username TEXT, @@ -3139,7 +3139,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get mapzen isochrones') finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( username TEXT, @@ -3200,7 +3200,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('Error trying to get Mapbox isochrones') finally: service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics @@ -3229,7 +3229,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- heremaps isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3244,7 +3244,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- mapzen isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3258,7 +3258,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- mapbox isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3272,7 +3272,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics @@ -3301,7 +3301,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- heremaps isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3316,7 +3316,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- mapzen isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3329,7 +3329,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; -- mapbox isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) @@ -3342,7 +3342,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +$$ LANGUAGE plpythonu STABLE ; DO $$ BEGIN IF NOT EXISTS ( From 50aa537a7a4e3993e0ef16186cf53be4a07201e8 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Mar 2018 16:13:56 +0200 Subject: [PATCH 042/265] Bump python library version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 7cc70f8..53d950d 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.5', + version='0.17.6', description='CartoDB Services API Python Library', From 5e141e3a10e9d25e1cf94db42cfebe374870e758 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 28 Mar 2018 10:34:45 +0200 Subject: [PATCH 043/265] Update NEWS.md --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index 3837af8..4b0b240 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ + +March 28th, 2018 +================ +* Version `0.30.5` of server side and `0.17.6` of the python library + * All the returned polygons are now always MULTIPOLYGON #488 + March 27th, 2018 ================ * Version `0.17.5` of python library From 6143c04c82433fb6985e9be0e2a28864e67a696c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Apr 2018 13:11:50 +0200 Subject: [PATCH 044/265] Remove server extension versioned files of outdated version changes --- ...db_dataservices_server--0.30.4--0.31.0.sql | 449 -- ...db_dataservices_server--0.31.0--0.30.4.sql | 175 - .../cdb_dataservices_server--0.31.0.sql | 3648 ----------------- 3 files changed, 4272 deletions(-) delete mode 100644 server/extension/cdb_dataservices_server--0.30.4--0.31.0.sql delete mode 100644 server/extension/cdb_dataservices_server--0.31.0--0.30.4.sql delete mode 100644 server/extension/cdb_dataservices_server--0.31.0.sql diff --git a/server/extension/cdb_dataservices_server--0.30.4--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.4--0.31.0.sql deleted file mode 100644 index 0b0d3cf..0000000 --- a/server/extension/cdb_dataservices_server--0.30.4--0.31.0.sql +++ /dev/null @@ -1,449 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.31.0'" to load this file. \quit - --- HERE goes your code to upgrade/downgrade -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT) -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomRouting - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.tools.polyline import polyline_to_linestring - from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if not waypoints or len(waypoints) < 2: - service_manager.logger.info("Empty origin or destination") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - if len(waypoints) > 25: - service_manager.logger.info("Too many waypoints (max 25)") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - waypoint_coords = [] - for waypoint in waypoints: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] - waypoint_coords.append(Coordinate(lon,lat)) - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - resp = client.directions(waypoint_coords, profile) - if resp and resp.shape: - shape_linestring = polyline_to_linestring(resp.shape) - if shape_linestring: - service_manager.quota_service.increment_success_service_use() - return [shape_linestring, resp.length, int(round(resp.duration))] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to calculate TomTom routing') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( - username TEXT, - orgname TEXT, - origin geometry(Point, 4326), - destination geometry(Point, 4326), - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_point', user_routing_config, logger): - waypoints = [origin, destination] - - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_waypoints', user_routing_config, logger): - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapzen_geocoder: - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapbox_geocoder: - mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.tomtom_geocoder: - tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Requested geocoder is not available') - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - # The configuration is retrieved but no checks are performed on it - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from iso3166 import countries - from cartodb_services.tools import ServiceManager, QuotaExceededException - from cartodb_services.tomtom import TomTomGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) - - try: - service_manager.assert_within_limits() - geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - country_iso3166 = None - if country: - country_iso3 = country_to_iso3(country) - if country_iso3: - country_iso3166 = countries.get(country_iso3).alpha2.lower() - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3166) - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using TomTom') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomIsolines - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - # -- TODO Support options properly - isolines = {} - for r in data_range: - isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) - isolines[r] = isoline - - result = [] - for r in data_range: - - if len(isolines[r]) >= 3: - # -- TODO encapsulate this block into a func/method - locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point - wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) - multipolygon = plpy.execute(sql, 1)[0]['geom'] - else: - multipolygon = None - - result.append([source, r, multipolygon]) - - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(isolines)) - return result - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get TomTom isolines') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomIsolines - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) - - if resp: - result = [] - for isochrone in resp: - result_polygon = coordinates_to_polygon(isochrone.coordinates) - if result_polygon: - service_manager.quota_service.increment_success_service_use() - result.append([source, isochrone.duration, result_polygon]) - else: - service_manager.quota_service.increment_empty_service_use() - result.append([source, isochrone.duration, None]) - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(result)) - return result - else: - service_manager.quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get TomTom isochrones') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isodistance', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- tomtom isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isochrone', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- tomtom isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.30.4.sql b/server/extension/cdb_dataservices_server--0.31.0--0.30.4.sql deleted file mode 100644 index f8d1aa7..0000000 --- a/server/extension/cdb_dataservices_server--0.31.0--0.30.4.sql +++ /dev/null @@ -1,175 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit - --- HERE goes your code to upgrade/downgrade -DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(TEXT, TEXT, geometry(Point, 4326)[], TEXT); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( - username TEXT, - orgname TEXT, - origin geometry(Point, 4326), - destination geometry(Point, 4326), - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_point', user_routing_config, logger): - waypoints = [origin, destination] - - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_waypoints', user_routing_config, logger): - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapzen_geocoder: - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapbox_geocoder: - mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Requested geocoder is not available') - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); - -DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); - -DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); - -DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isochrones(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isodistance', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isochrone', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql deleted file mode 100644 index 543ea77..0000000 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ /dev/null @@ -1,3648 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit -CREATE TYPE cdb_dataservices_server.simple_route AS ( - shape geometry(LineString,4326), - length real, - duration integer -); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT) -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.mapbox import MapboxRouting - from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.tools import Coordinate - from cartodb_services.tools.polyline import polyline_to_linestring - from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - - if not waypoints or len(waypoints) < 2: - service_manager.logger.info("Empty origin or destination") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - if len(waypoints) > 25: - service_manager.logger.info("Too many waypoints (max 25)") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - waypoint_coords = [] - for waypoint in waypoints: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] - waypoint_coords.append(Coordinate(lon,lat)) - - profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) - - resp = client.directions(waypoint_coords, profile) - if resp and resp.shape: - shape_linestring = polyline_to_linestring(resp.shape) - if shape_linestring: - service_manager.quota_service.increment_success_service_use() - return [shape_linestring, resp.length, int(round(resp.duration))] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to calculate Mapbox routing') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT) -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomRouting - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.tools.polyline import polyline_to_linestring - from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if not waypoints or len(waypoints) < 2: - service_manager.logger.info("Empty origin or destination") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - if len(waypoints) > 25: - service_manager.logger.info("Too many waypoints (max 25)") - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - - waypoint_coords = [] - for waypoint in waypoints: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] - waypoint_coords.append(Coordinate(lon,lat)) - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - resp = client.directions(waypoint_coords, profile) - if resp and resp.shape: - shape_linestring = polyline_to_linestring(resp.shape) - if shape_linestring: - service_manager.quota_service.increment_success_service_use() - return [shape_linestring, resp.length, int(round(resp.duration))] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - else: - service_manager.quota_service.increment_empty_service_use() - return [None, None, None] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to calculate TomTom routing') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - import json - from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse - from cartodb_services.mapzen.types import polyline_to_linestring - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Coordinate - from cartodb_services.tools import Logger,LoggerConfig - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - quota_service = QuotaService(user_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - try: - client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) - - if not waypoints or len(waypoints) < 2: - logger.info("Empty origin or destination") - quota_service.increment_empty_service_use() - return [None, None, None] - - waypoint_coords = [] - for waypoint in waypoints: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] - waypoint_coords.append(Coordinate(lon,lat)) - - resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) - if resp and resp.shape: - shape_linestring = polyline_to_linestring(resp.shape) - if shape_linestring: - quota_service.increment_success_service_use() - return [shape_linestring, resp.length, resp.duration] - else: - quota_service.increment_empty_service_use() - return [None, None, None] - else: - quota_service.increment_empty_service_use() - return [None, None, None] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to calculate mapzen routing') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( - username TEXT, - orgname TEXT, - origin geometry(Point, 4326), - destination geometry(Point, 4326), - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_point', user_routing_config, logger): - waypoints = [origin, destination] - - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( - username TEXT, - orgname TEXT, - waypoints geometry(Point, 4326)[], - mode TEXT, - options text[] DEFAULT ARRAY[]::text[], - units text DEFAULT 'kilometers') -RETURNS cdb_dataservices_server.simple_route AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_route_with_waypoints', user_routing_config, logger): - if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] - else: - raise Exception('Requested routing method is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; --- Get the connection to redis from cache or create a new one -CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) -RETURNS boolean AS $$ - cache_key = "redis_connection_{0}".format(user_id) - if cache_key in GD: - return False - else: - from cartodb_services.tools import RedisConnection, RedisDBConfig - metadata_config = RedisDBConfig('redis_metadata_config', plpy) - metrics_config = RedisDBConfig('redis_metrics_config', plpy) - redis_metadata_connection = RedisConnection(metadata_config).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config).redis_connection() - GD[cache_key] = { - 'redis_metadata_connection': redis_metadata_connection, - 'redis_metrics_connection': redis_metrics_connection, - } - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; --- --- Observatory connection config --- --- The purpose of this function is provide to the PL/Proxy functions --- the connection string needed to connect with the current production database - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( - username TEXT, - orgname TEXT) -RETURNS text AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] - - return user_obs_config.connection_str -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL, - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL, - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - import json - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getdemographicsnapshot', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL, - geometry_level TEXT DEFAULT NULL) -RETURNS SETOF json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL, - geometry_level TEXT DEFAULT NULL) -RETURNS SETOF JSON AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getdemographicsnapshot', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - import json - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getsegmentsnapshot', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_segment_snapshot') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT NULL) -RETURNS SETOF json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT NULL) -RETURNS SETOF JSON AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getsegmentsnapshot', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetSegmentSnapshot') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - measure_id TEXT, - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - measure_id TEXT, - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getmeasure', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasure') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - category_id TEXT, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - category_id TEXT, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getcategory', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetCategory') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - name TEXT, - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - name TEXT, - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getuscensusmeasure', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusMeasure') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - name TEXT, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - name TEXT, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getuscensuscategory', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusCategory') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - normalize TEXT DEFAULT NULL, - boundary_id TEXT DEFAULT NULL, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getpopulation', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['population'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPopulation') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( - username TEXT, - orgname TEXT, - geom_ref TEXT, - measure_id TEXT, - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( - username TEXT, - orgname TEXT, - geom_ref TEXT, - measure_id TEXT, - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS NUMERIC AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getmeasurebyid', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( - username TEXT, - orgname TEXT, - geomvals geomval[], - params JSON, - merge BOOLEAN DEFAULT True) -RETURNS TABLE ( - id INT, - data JSON -) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( - username TEXT, - orgname TEXT, - geomvals geomval[], - params JSON, - merge BOOLEAN DEFAULT True) -RETURNS TABLE ( - id INT, - data JSON -) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getdata', user_obs_config, logger, params): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) - result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) - empty_results = len(geomvals) - len(result) - if empty_results > 0: - quota_service.increment_empty_service_use(empty_results) - if result: - quota_service.increment_success_service_use(len(result)) - return result - else: - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use(len(geomvals)) - logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetData') - finally: - quota_service.increment_total_service_use(len(geomvals)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( - username TEXT, - orgname TEXT, - geomrefs TEXT[], - params JSON) -RETURNS TABLE ( - id TEXT, - data JSON -) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( - username TEXT, - orgname TEXT, - geomrefs TEXT[], - params JSON) -RETURNS TABLE ( - id TEXT, - data JSON -) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getdata', user_obs_config, logger, params): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) - result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) - empty_results = len(geomrefs) - len(result) - if empty_results > 0: - quota_service.increment_empty_service_use(empty_results) - if result: - quota_service.increment_success_service_use(len(result)) - return result - else: - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use(len(geomrefs)) - exc_info = sys.exc_info() - logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) - logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetData') - finally: - quota_service.increment_total_service_use(len(geomrefs)) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( - username TEXT, - orgname TEXT, - geom Geometry(Geometry, 4326), - params JSON, - max_timespan_rank INTEGER DEFAULT NULL, - max_score_rank INTEGER DEFAULT NULL, - target_geoms INTEGER DEFAULT NULL) -RETURNS JSON AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( - username TEXT, - orgname TEXT, - geom Geometry(Geometry, 4326), - params JSON, - max_timespan_rank INTEGER DEFAULT NULL, - max_score_rank INTEGER DEFAULT NULL, - target_geoms INTEGER DEFAULT NULL) -RETURNS JSON AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('obs_getmeta', user_obs_config, logger, params): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) - result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) - if result: - return result[0]['meta'] - else: - return None - except BaseException as e: - import sys - logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeta') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( - username TEXT, - orgname TEXT, - geometry_extent Geometry(Geometry, 4326), - geometry_type text, - params JSON, - target_geoms INTEGER DEFAULT NULL) -RETURNS TABLE(valid boolean, errors text[]) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( - username TEXT, - orgname TEXT, - geometry_extent Geometry(Geometry, 4326), - geometry_type text, - params JSON, - target_geoms INTEGER DEFAULT NULL) -RETURNS TABLE(valid boolean, errors text[]) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('obs_metadatavalidation', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) - result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) - if result: - return result - else: - return [] - except BaseException as e: - import sys - logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_MetadataValidation') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( - username TEXT, - orgname TEXT, - search_term TEXT, - relevant_boundary TEXT DEFAULT NULL) -RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( - username TEXT, - orgname TEXT, - search_term TEXT, - relevant_boundary TEXT DEFAULT NULL) -RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_search', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) - if result: - resp = [] - for element in result: - id = element['id'] - description = element['description'] - name = element['name'] - aggregate = element['aggregate'] - source = element['source'] - resp.append([id, description, name, aggregate, source]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [None, None, None, None, None] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_Search') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL) -RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL) -RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getavailableboundaries', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) - if result: - resp = [] - for element in result: - id = element['boundary_id'] - description = element['description'] - tspan = element['time_span'] - tablename = element['tablename'] - resp.append([id, description, tspan, tablename]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS geometry(Geometry, 4326) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS geometry(Geometry, 4326) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getboundary', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundary') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS TEXT AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getboundaryid', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryId') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( - username TEXT, - orgname TEXT, - geometry_id TEXT, - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS geometry(Geometry, 4326) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( - username TEXT, - orgname TEXT, - geometry_id TEXT, - boundary_id TEXT, - time_span TEXT DEFAULT NULL) -RETURNS geometry(Geometry, 4326) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getboundarybyid', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryById') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type text DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getboundariesbygeometry', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByGeometry') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - radius NUMERIC, - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - radius NUMERIC, - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getpointsbygeometry', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByGeometry') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - radius NUMERIC, - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( - username TEXT, - orgname TEXT, - geom geometry(Point, 4326), - radius NUMERIC, - boundary_id TEXT, - time_span TEXT DEFAULT NULL, - overlap_type TEXT DEFAULT NULL) -RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_obs_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('obs_getpointsbypointandradius', user_obs_config, logger): - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByPointAndRadius') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); - -CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) -RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ - host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] - return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) - )[0] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) -RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_ConnectUserTable; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) -RETURNS cdb_dataservices_server.ds_return_metadata AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_GetReturnMetadata; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) -RETURNS SETOF record AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_FetchJoinFdwTableData; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) -RETURNS boolean AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_DisconnectUserTable; -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) -RETURNS text AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.obs_dumpversion(); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - --- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them -CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( - username TEXT, - orgname TEXT, - bounds geometry(Geometry, 4326) DEFAULT NULL, - filter_tags TEXT[] DEFAULT NULL, - denom_id TEXT DEFAULT NULL, - geom_id TEXT DEFAULT NULL, - timespan TEXT DEFAULT NULL) -RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( - username TEXT, - orgname TEXT, - bounds geometry(Geometry, 4326) DEFAULT NULL, - section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], - subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], - other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], - ids TEXT[] DEFAULT ARRAY[]::TEXT[], - name TEXT DEFAULT NULL, - denom_id TEXT DEFAULT '', - geom_id TEXT DEFAULT '', - timespan TEXT DEFAULT '') -RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( - username TEXT, - orgname TEXT, - bounds geometry(Geometry, 4326) DEFAULT NULL, - filter_tags TEXT[] DEFAULT NULL, - numer_id TEXT DEFAULT NULL, - geom_id TEXT DEFAULT NULL, - timespan TEXT DEFAULT NULL) -RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( - username TEXT, - orgname TEXT, - bounds geometry(Geometry, 4326) DEFAULT NULL, - filter_tags TEXT[] DEFAULT NULL, - numer_id TEXT DEFAULT NULL, - denom_id TEXT DEFAULT NULL, - timespan TEXT DEFAULT NULL, - number_geometries INTEGER DEFAULT NULL) -RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( - username TEXT, - orgname TEXT, - bounds geometry(Geometry, 4326) DEFAULT NULL, - filter_tags TEXT[] DEFAULT NULL, - numer_id TEXT DEFAULT NULL, - denom_id TEXT DEFAULT NULL, - geom_id TEXT DEFAULT NULL) -RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( - username TEXT, - orgname TEXT, - aggregate_type TEXT DEFAULT NULL) -RETURNS TABLE(name TEXT, subsection JSON) AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() -RETURNS boolean AS $$ - cache_key = "logger_config" - if cache_key in GD: - return False - else: - from cartodb_services.tools import LoggerConfig - logger_config = LoggerConfig(plpy) - GD[cache_key] = logger_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - --- This is done in order to avoid an undesired depedency on cartodb extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) -RETURNS JSON AS $$ - SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; -$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; - -CREATE OR REPLACE -FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) - RETURNS void AS $$ -BEGIN - PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); - EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; -END -$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE -FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) - RETURNS void AS $$ -BEGIN - EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; -END -$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) -RETURNS boolean AS $$ - cache_key = "user_geocoder_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import GeocoderConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) - GD[cache_key] = geocoder_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_internal_geocoder_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import InternalGeocoderConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = geocoder_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_isolines_routing_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import IsolinesRoutingConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = isolines_routing_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_routing_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import RoutingConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - routing_config = RoutingConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = routing_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_obs_snapshot_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import ObservatorySnapshotConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = obs_snapshot_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_obs_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import ObservatoryConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = obs_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) - WHERE pg_type.typname = 'service_type' - AND pg_namespace.nspname = 'cdb_dataservices_server') THEN - CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( - 'isolines', - 'hires_geocoder', - 'routing', - 'observatory' - ); - END IF; -END $$; - -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) - WHERE pg_type.typname = 'service_quota_info' - AND pg_namespace.nspname = 'cdb_dataservices_server') THEN - CREATE TYPE cdb_dataservices_server.service_quota_info AS ( - service cdb_dataservices_server.service_type, - monthly_quota NUMERIC, - used_quota NUMERIC, - soft_limit BOOLEAN, - provider TEXT - ); - END IF; -END $$; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( - username TEXT, - orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ - from cartodb_services.metrics.user import UserMetricsService - from datetime import date - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - today = date.today() - ret = [] - - #-- Isolines - service = 'isolines' - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) - - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Hires Geocoder - service = 'hires_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Routing - service = 'routing' - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_routing_config, redis_conn) - - monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config.service_type, today) - soft_limit = user_routing_config.soft_limit - provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Observatory - service = 'observatory' - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - user_service = UserMetricsService(user_obs_config, redis_conn) - - monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config.service_type, today) - soft_limit = user_obs_config.soft_limit - provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( - username TEXT, - orgname TEXT, - service_ TEXT, - input_size NUMERIC) -returns BOOLEAN AS $$ - DECLARE - params cdb_dataservices_server.service_quota_info; - BEGIN - SELECT * INTO params - FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p - WHERE p.service = service_::cdb_dataservices_server.service_type; - RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); - END -$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; --- Geocodes a street address given a searchtext and a state and/or country - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapzen_geocoder: - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapbox_geocoder: - mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.tomtom_geocoder: - tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Requested geocoder is not available') - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Here geocoder is not available for your account.') - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - if user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Google geocoder is not available for your account.') - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - # The configuration is retrieved but no checks are performed on it - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - # The configuration is retrieved but no checks are performed on it - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - # The configuration is retrieved but no checks are performed on it - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.tools import LegacyServiceManager - from cartodb_services.tools import QuotaExceededException - from cartodb_services.here import HereMapsGeocoder - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - - try: - service_manager.assert_within_limits() - geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) - coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using here maps') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.tools import LegacyServiceManager, QuotaExceededException - from cartodb_services.google import GoogleMapsGeocoder - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - - try: - service_manager.assert_within_limits(quota=False) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) - coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using google maps') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.tools import ServiceManager, QuotaExceededException - from cartodb_services.mapzen import MapzenGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) - - try: - service_manager.assert_within_limits() - geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) - country_iso3 = None - if country: - country_iso3 = country_to_iso3(country) - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3, search_type='address') - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using mapzen') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from iso3166 import countries - from cartodb_services.tools import ServiceManager, QuotaExceededException - from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) - - try: - service_manager.assert_within_limits() - geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - - country_iso3166 = None - if country: - country_iso3 = country_to_iso3(country) - if country_iso3: - country_iso3166 = countries.get(country_iso3).alpha2.lower() - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3166) - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using Mapbox') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from iso3166 import countries - from cartodb_services.tools import ServiceManager, QuotaExceededException - from cartodb_services.tomtom import TomTomGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) - - try: - service_manager.assert_within_limits() - geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - country_iso3166 = None - if country: - country_iso3 = country_to_iso3(country) - if country_iso3: - country_iso3166 = countries.get(country_iso3).alpha2.lower() - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3166) - if coordinates: - service_manager.quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - service_manager.quota_service.increment_empty_service_use() - return None - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use() - return None - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using TomTom') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS JSON AS $$ - import json - from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_config = ServiceConfiguration(service, username, orgname) - rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() - if rate_limit_config.is_limited(): - return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) - else: - return None -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( - username TEXT, - orgname TEXT, - service TEXT, - rate_limit_json JSON) -RETURNS VOID AS $$ - import json - from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter - - import cartodb_services - cartodb_services.init(plpy, GD) - - config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) - if rate_limit_json: - rate_limit = json.loads(rate_limit_json) - limit = rate_limit.get('limit', None) - period = rate_limit.get('period', None) - else: - limit = None - period = None - config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) - config_setter.set_user_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( - username TEXT, - orgname TEXT, - service TEXT, - rate_limit_json JSON) -RETURNS VOID AS $$ - import json - from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter - - import cartodb_services - cartodb_services.init(plpy, GD) - - config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) - if rate_limit_json: - rate_limit = json.loads(rate_limit_json) - limit = rate_limit.get('limit', None) - period = rate_limit.get('period', None) - else: - limit = None - period = None - config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) - config_setter.set_org_rate_limits(config) -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( - username TEXT, - orgname TEXT, - service TEXT, - rate_limit_json JSON) -RETURNS VOID AS $$ - import json - from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter - - import cartodb_services - cartodb_services.init(plpy, GD) - - config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) - if rate_limit_json: - rate_limit = json.loads(rate_limit_json) - limit = rate_limit.get('limit', None) - period = rate_limit.get('period', None) - else: - limit = None - period = None - config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) - config_setter.set_server_rate_limits(config) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin0 polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT n.the_geom as geom INTO ret - FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x - FROM (SELECT country_name q) g) d - LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x - LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [admin1_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin1 polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) - rv = plpy.execute(plan, [admin1_name, country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin1 polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT q, ( - SELECT the_geom - FROM global_province_polygons - WHERE d.c = ANY (synonyms) - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM ( - SELECT - trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q - ) d - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) - SELECT - geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_province_polygons - WHERE p.c = ANY (synonyms) - AND iso3 = p.i - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM p) n; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) -RETURNS Geometry AS $$ - import spiexceptions - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] - except spiexceptions.ExternalRoutineException as e: - import sys - logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) -RETURNS Geometry AS $$ - import spiexceptions - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] - except spiexceptions.ExternalRoutineException as e: - import sys - logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - import spiexceptions - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] - except spiexceptions.ExternalRoutineException as e: - import sys - logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig, metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - if admin1_name and country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) - rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) - elif country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) - rv = plpy.execute(plan, [city_name, country_name], 1) - else: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [city_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode namedplace point') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), - next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), - best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH inputcountry AS ( - SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 - ), - p AS ( - SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), - best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) -RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) -RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) -RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; - - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) -RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); -$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) - rv = plpy.execute(plan, [ip], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - - new_ip INET; - BEGIN - BEGIN - IF family(ip::inet) = 6 THEN - new_ip := ip::inet; - ELSE - new_ip := ('::ffff:' || ip)::inet; - END IF; - EXCEPTION WHEN OTHERS THEN - SELECT NULL as geom INTO ret; - RETURN ret; - END; - - WITH - ips AS (SELECT ip s, new_ip net), - matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) - SELECT geom INTO ret - FROM matches; - RETURN ret; -END -$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; -CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json - from cartodb_services.here import HereMapsRoutingIsoline - from cartodb_services.metrics import QuotaService - from cartodb_services.here.types import geo_polyline_to_multipolygon - from cartodb_services.tools import Logger,LoggerConfig - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - try: - client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, - user_isolines_routing_config.heremaps_app_code, logger) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - source_str = 'geo!%f,%f' % (lat, lon) - else: - source_str = None - - if type == 'isodistance': - resp = client.calculate_isodistance(source_str, mode, data_range, options) - elif type == 'isochrone': - resp = client.calculate_isochrone(source_str, mode, data_range, options) - - if resp: - result = [] - for isoline in resp: - data_range_n = isoline['range'] - polyline = isoline['geom'] - multipolygon = geo_polyline_to_multipolygon(polyline) - result.append([source, data_range_n, multipolygon]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(resp)) - return result - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get mapzen isolines') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json - from cartodb_services.mapzen import MatrixClient, MapzenIsolines - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - try: - client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) - mapzen_isolines = MapzenIsolines(client, logger) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = {'lat': lat, 'lon': lon} - else: - raise Exception('source is NULL') - - # -- TODO Support options properly - isolines = {} - for r in data_range: - isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) - isolines[r] = isoline - - result = [] - for r in data_range: - - if len(isolines[r]) >= 3: - # -- TODO encapsulate this block into a func/method - locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point - wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) - sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) - multipolygon = plpy.execute(sql, 1)[0]['geom'] - else: - multipolygon = None - - result.append([source, r, multipolygon]) - - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(isolines)) - return result - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get mapzen isolines') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines - from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.tools import Coordinate - from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - mapbox_isolines = MapboxIsolines(client, service_manager.logger) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) - - # -- TODO Support options properly - isolines = {} - for r in data_range: - isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) - isolines[r] = isoline - - result = [] - for r in data_range: - - if len(isolines[r]) >= 3: - # -- TODO encapsulate this block into a func/method - locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point - wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) - multipolygon = plpy.execute(sql, 1)[0]['geom'] - else: - multipolygon = None - - result.append([source, r, multipolygon]) - - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(isolines)) - return result - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get Mapbox isolines') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomIsolines - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - # -- TODO Support options properly - isolines = {} - for r in data_range: - isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) - isolines[r] = isoline - - result = [] - for r in data_range: - - if len(isolines[r]) >= 3: - # -- TODO encapsulate this block into a func/method - locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point - wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) - sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) - multipolygon = plpy.execute(sql, 1)[0]['geom'] - else: - multipolygon = None - - result.append([source, r, multipolygon]) - - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(isolines)) - return result - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get TomTom isolines') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json - from cartodb_services.mapzen import MatrixClient, MapzenIsochrones - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - from cartodb_services.mapzen.types import coordinates_to_polygon - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - try: - mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, - logger, user_isolines_routing_config.mapzen_isochrones_service_params) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = {'lat': lat, 'lon': lon} - else: - raise Exception('source is NULL') - - resp = mapzen_isochrones.isochrone(origin, mode, data_range) - - if resp: - result = [] - for isochrone in resp: - result_polygon = coordinates_to_polygon(isochrone.coordinates) - if result_polygon: - quota_service.increment_success_service_use() - result.append([source, isochrone.duration, result_polygon]) - else: - quota_service.increment_empty_service_use() - result.append([source, isochrone.duration, None]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(result)) - return result - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get mapzen isochrones') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines - from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.tools import Coordinate - from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - mapbox_isolines = MapboxIsolines(client, service_manager.logger) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) - - resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) - - if resp: - result = [] - for isochrone in resp: - result_polygon = coordinates_to_polygon(isochrone.coordinates) - if result_polygon: - service_manager.quota_service.increment_success_service_use() - result.append([source, isochrone.duration, result_polygon]) - else: - service_manager.quota_service.increment_empty_service_use() - result.append([source, isochrone.duration, None]) - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(result)) - return result - else: - service_manager.quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get Mapbox isochrones') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( - username TEXT, - orgname TEXT, - source geometry(Geometry, 4326), - mode TEXT, - data_range integer[], - options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.tools import ServiceManager - from cartodb_services.tomtom import TomTomIsolines - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM - from cartodb_services.tools import Coordinate - from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder - - import cartodb_services - cartodb_services.init(plpy, GD) - - service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) - service_manager.assert_within_limits() - - try: - tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - origin = Coordinate(lon,lat) - else: - raise Exception('source is NULL') - - profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) - - resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) - - if resp: - result = [] - for isochrone in resp: - result_polygon = coordinates_to_polygon(isochrone.coordinates) - if result_polygon: - service_manager.quota_service.increment_success_service_use() - result.append([source, isochrone.duration, result_polygon]) - else: - service_manager.quota_service.increment_empty_service_use() - result.append([source, isochrone.duration, None]) - service_manager.quota_service.increment_success_service_use() - service_manager.quota_service.increment_isolines_service_use(len(result)) - return result - else: - service_manager.quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to get TomTom isochrones') - finally: - service_manager.quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isodistance', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- heremaps isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - type = 'isodistance' - - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- mapzen isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- mapbox isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- tomtom isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - from cartodb_services.metrics import metrics - from cartodb_services.tools import Logger - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - if user_isolines_config.google_services_user: - raise Exception('This service is not available for google service users.') - - with metrics('cb_isochrone', user_isolines_config, logger): - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- heremaps isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - type = 'isochrone' - - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) - - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- mapzen isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- mapbox isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - --- tomtom isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) - return result -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT * - FROM pg_catalog.pg_user - WHERE usename = 'geocoder_api') THEN - - CREATE USER geocoder_api; - END IF; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; - GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; - GRANT USAGE ON SCHEMA public TO geocoder_api; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; -END$$; From adde2d3a46ebc7017501aaa29789b9d886cff8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Apr 2018 14:41:44 +0200 Subject: [PATCH 045/265] 0.31.0 release scripts --- ...db_dataservices_server--0.30.5--0.31.0.sql | 475 +++ ...db_dataservices_server--0.31.0--0.30.5.sql | 182 + .../cdb_dataservices_server--0.31.0.sql | 3648 +++++++++++++++++ ...db_dataservices_server--0.30.4--0.30.5.sql | 0 ...db_dataservices_server--0.30.5--0.30.4.sql | 0 .../cdb_dataservices_server--0.30.5.sql | 0 6 files changed, 4305 insertions(+) create mode 100644 server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql create mode 100644 server/extension/cdb_dataservices_server--0.31.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.4--0.30.5.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.5--0.30.4.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.5.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql new file mode 100644 index 0000000..e1a25c8 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql @@ -0,0 +1,475 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.31.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql b/server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql new file mode 100644 index 0000000..afbd6ad --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql @@ -0,0 +1,182 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.5'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(TEXT, TEXT, geometry(Point, 4326)[], TEXT); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE ; + + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isochrones(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE ; + + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE ; + + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql new file mode 100644 index 0000000..7215d78 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -0,0 +1,3648 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.5.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.4--0.30.5.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.5.sql diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.5--0.30.4.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.5--0.30.4.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.5--0.30.4.sql diff --git a/server/extension/cdb_dataservices_server--0.30.5.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.5.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.5.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.5.sql From e21d3e2e70745e3cd4bdfac447753c5b98b674e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Apr 2018 16:29:00 +0200 Subject: [PATCH 046/265] TomTom Python lib version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 844be14..0dc3917 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.4', + version='0.17.7', description='CartoDB Services API Python Library', From 94aeceb8944e709e66754c33a913204d20ae63c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Apr 2018 19:04:24 +0200 Subject: [PATCH 047/265] Fixes encoding --- .../cartodb_services/tomtom/geocoder.py | 12 ++++++++++++ .../cartodb_services/test/test_tomtomgeocoder.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 783be25..432dd31 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -1,3 +1,6 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + import json import requests from uritemplate import URITemplate @@ -62,6 +65,15 @@ class TomTomGeocoder(Traceable): @qps_retry(qps=5) def geocode(self, searchtext, city=None, state_province=None, country=None): + if searchtext: + searchtext = searchtext.decode('utf-8') + if city: + city = city.decode('utf-8') + if state_province: + state_province = state_province.decode('utf-8') + if country: + country = country.decode('utf-8') + if not self._validate_input(searchtext, city, state_province, country): return [] diff --git a/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py index 7fcfcda..002847a 100644 --- a/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_tomtomgeocoder.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import unittest from mock import Mock from cartodb_services.tomtom import TomTomGeocoder @@ -5,7 +7,7 @@ from cartodb_services.tools.exceptions import ServiceException from credentials import tomtom_api_key INVALID_APIKEY = 'invalid_apikey' -VALID_ADDRESS = 'Plaza Mayor 3, Valladolid' +VALID_ADDRESS = u'Mantería 3, Valladolid'.encode('utf-8') class TomTomGeocoderTestCase(unittest.TestCase): From 33f23e0902cadf87f0b2d4b0c7a73504d20eaab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Apr 2018 11:36:34 +0200 Subject: [PATCH 048/265] Minor instead of patch for TomTom support at Python lib --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 0dc3917..61c8a3c 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.17.7', + version='0.18.0', description='CartoDB Services API Python Library', From e5d2182da30dbba520af3b26457dcfb2fa9c4866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Apr 2018 11:50:30 +0200 Subject: [PATCH 049/265] Removed geocoder_api user creation from version upgrade script --- .../cdb_dataservices_server--0.30.5--0.31.0.sql | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql index e1a25c8..73d095e 100644 --- a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql @@ -458,18 +458,3 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) return result $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT * - FROM pg_catalog.pg_user - WHERE usename = 'geocoder_api') THEN - - CREATE USER geocoder_api; - END IF; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; - GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; - GRANT USAGE ON SCHEMA public TO geocoder_api; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; -END$$; \ No newline at end of file From c3c28dfd5ea457734bd5e6f973708ab2202725b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Apr 2018 12:14:11 +0200 Subject: [PATCH 050/265] Simpler quota check matching --- .../cartodb_services/metrics/quota.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 07a4c2d..6b3ca3b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -69,31 +69,16 @@ class QuotaChecker: def check(self): """ Check if the current user quota surpasses the current quota """ - if re.match('geocoder_*', + if re.match('^geocoder_', self._user_service_config.service_type) is not None: return self.__check_geocoder_quota() - elif re.match('here_isolines', + elif re.match('_isolines$', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() - elif re.match('mapzen_isolines', - self._user_service_config.service_type) is not None: - return self.__check_isolines_quota() - elif re.match('mapbox_isolines', - self._user_service_config.service_type) is not None: - return self.__check_isolines_quota() - elif re.match('tomtom_isolines', - self._user_service_config.service_type) is not None: - return self.__check_isolines_quota() - elif re.match('routing_mapzen', + elif re.match('^routing_', self._user_service_config.service_type) is not None: return self.__check_routing_quota() - elif re.match('routing_mapbox', - self._user_service_config.service_type) is not None: - return self.__check_routing_quota() - elif re.match('routing_tomtom', - self._user_service_config.service_type) is not None: - return self.__check_routing_quota() - elif re.match('obs_*', + elif re.match('^obs_', self._user_service_config.service_type) is not None: return self.__check_data_observatory_quota() else: From 193513bfea577133244d1d1aacedd4ef0f9a1368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Apr 2018 12:56:55 +0200 Subject: [PATCH 051/265] params at metrics --- .../cdb_dataservices_server--0.30.5--0.31.0.sql | 12 +++++++++--- server/extension/cdb_dataservices_server--0.31.0.sql | 12 +++++++++--- server/extension/sql/20_geocode_street.sql | 4 +++- server/extension/sql/85_isodistance.sql | 4 +++- server/extension/sql/90_isochrone.sql | 4 +++- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql index 73d095e..040af68 100644 --- a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql @@ -154,7 +154,9 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.heremaps_geocoder: here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] @@ -379,7 +381,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isodistance', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) @@ -428,7 +432,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isochrone', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql index 7215d78..a62e503 100644 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -1956,7 +1956,9 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.heremaps_geocoder: here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] @@ -3472,7 +3474,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isodistance', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) @@ -3561,7 +3565,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isochrone', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 6466855..30a477a 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -12,7 +12,9 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.heremaps_geocoder: here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 18bd35f..9609c27 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -14,7 +14,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isodistance', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 8e5721f..61a5fce 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -14,7 +14,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isochrone', user_isolines_config, logger): + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) From 7577936c33065dbdd456f5f3d44a192e39481bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Apr 2018 15:24:17 +0200 Subject: [PATCH 052/265] Fixes isolines quota check --- .../python/cartodb_services/cartodb_services/metrics/quota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 6b3ca3b..476318c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -72,7 +72,7 @@ class QuotaChecker: if re.match('^geocoder_', self._user_service_config.service_type) is not None: return self.__check_geocoder_quota() - elif re.match('_isolines$', + elif re.match('.*_isolines$', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() elif re.match('^routing_', From 3a240bf6ad40631d47bceb697dce5e9081886430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 27 Apr 2018 12:43:29 +0200 Subject: [PATCH 053/265] params at routing metrics --- .../extension/cdb_dataservices_server--0.30.5--0.31.0.sql | 8 ++++++-- server/extension/cdb_dataservices_server--0.31.0.sql | 8 ++++++-- server/extension/sql/105_route_between_points.sql | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql index 040af68..cf67925 100644 --- a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql @@ -85,7 +85,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_point', user_routing_config, logger): + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): waypoints = [origin, destination] if user_routing_config.mapzen_provider: @@ -124,7 +126,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_waypoints', user_routing_config, logger): + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql index a62e503..2e7b27c 100644 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -209,7 +209,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_point', user_routing_config, logger): + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): waypoints = [origin, destination] if user_routing_config.mapzen_provider: @@ -248,7 +250,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_waypoints', user_routing_config, logger): + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 841ef39..87769c9 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -18,7 +18,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_point', user_routing_config, logger): + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): waypoints = [origin, destination] if user_routing_config.mapzen_provider: @@ -57,7 +59,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - with metrics('cdb_route_with_waypoints', user_routing_config, logger): + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) From 4b599ecf78eab02603d3c99819522d29746ace47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 7 May 2018 11:08:51 +0200 Subject: [PATCH 054/265] NEWS --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 4b0b240..85db02d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +May 7th, 2018 +============= +* Version `0.24.0` of the client, `0.31.0` of the server, and ' 0.18.0` of the python library. + * Support for TomTom routing, geocoding and isolines. March 28th, 2018 ================ From 9c428dbf31c8fafba452b8175018af4faf68f6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 7 May 2018 11:17:43 +0200 Subject: [PATCH 055/265] Fix quote at NEWS --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 85db02d..786d41d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ May 7th, 2018 ============= -* Version `0.24.0` of the client, `0.31.0` of the server, and ' 0.18.0` of the python library. +* Version `0.24.0` of the client, `0.31.0` of the server, and `0.18.0` of the python library. * Support for TomTom routing, geocoding and isolines. March 28th, 2018 From 6b117e26b15737f839bf58d4140726816f1edae7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 May 2018 12:22:39 +0200 Subject: [PATCH 056/265] Remove unused Mapzen test --- test/integration/test_street_functions.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 47dd2b4..a409864 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -23,14 +23,6 @@ class TestStreetFunctions(TestCase): geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) - def test_if_select_with_mapzen_provider_street_point_is_ok(self): - query = "SELECT cdb_mapzen_geocode_street_point(street) " \ - "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) - geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) - assert_not_equal(geometry['geometry'], None) - def test_if_select_with_street_without_api_key_raise_error(self): query = "SELECT cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1".format( From 5251534283d233d5f1e9cfc33f8de9cf18cd3ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 4 Jun 2018 18:07:16 +0200 Subject: [PATCH 057/265] Allow using non-Premium keys for Google Maps client --- .../cartodb_services/google/client_factory.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py index feb7f4a..9593f4f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/client_factory.py @@ -5,6 +5,7 @@ import googlemaps import base64 from exceptions import InvalidGoogleCredentials + class GoogleMapsClientFactory(): clients = {} @@ -13,11 +14,14 @@ class GoogleMapsClientFactory(): cache_key = "{}:{}:{}".format(client_id, client_secret, channel) client = cls.clients.get(cache_key) if not client: - cls.assert_valid_crendentials(client_secret) - client = googlemaps.Client( - client_id=client_id, - client_secret=client_secret, - channel=channel) + if client_id: + cls.assert_valid_crendentials(client_secret) + client = googlemaps.Client( + client_id=client_id, + client_secret=client_secret, + channel=channel) + else: + client = googlemaps.Client(key=client_secret) cls.clients[cache_key] = client return client From be446c1bf26ce0f399168f4c3413c6903318ade7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 7 Jun 2018 10:29:57 +0200 Subject: [PATCH 058/265] exception logging at _send_to_plpy --- .../cartodb_services/tools/log.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index 40e3eec..817a423 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -35,28 +35,28 @@ class Logger: return self._send_to_rollbar('debug', text, exception, data) self._send_to_log_file('debug', text, exception, data) - self._send_to_plpy('debug', text) + self._send_to_plpy('debug', text, exception) def info(self, text, exception=None, data={}): if not self._check_min_level('info'): return self._send_to_rollbar('info', text, exception, data) self._send_to_log_file('info', text, exception, data) - self._send_to_plpy('info', text) + self._send_to_plpy('info', text, exception) def warning(self, text, exception=None, data={}): if not self._check_min_level('warning'): return self._send_to_rollbar('warning', text, exception, data) self._send_to_log_file('warning', text, exception, data) - self._send_to_plpy('warning', text) + self._send_to_plpy('warning', text, exception) def error(self, text, exception=None, data={}): if not self._check_min_level('error'): return self._send_to_rollbar('error', text, exception, data) self._send_to_log_file('error', text, exception, data) - self._send_to_plpy('error', text) + self._send_to_plpy('error', text, exception) def _check_min_level(self, level): return True if self.LEVELS[level] >= self._min_level else False @@ -85,18 +85,20 @@ class Logger: elif level == 'error': self._file_logger.error(text, extra=extra_data) - def _send_to_plpy(self, level, text): + def _send_to_plpy(self, level, text, exception=None): + exception_message = '. Exception: {}'.format(str(exception)) if exception else '' + message = '{}{}'.format(text, exception_message) if self._check_plpy(): if level == 'debug': - plpy.debug(text) + plpy.debug(message) elif level == 'info': - plpy.info(text) + plpy.info(message) elif level == 'warning': - plpy.warning(text) + plpy.warning(message) elif level == 'error': # Plpy.error and fatal raises exceptions and we only want to # log an error, exceptions should be raise explicitly - plpy.warning(text) + plpy.warning(message) def _parse_log_extra_data(self, exception, data): extra_data = {} From 3f08d37ef7de2745dd98bd337d6a4a5cfb5542fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 7 Jun 2018 14:35:51 +0200 Subject: [PATCH 059/265] Google bulk_geocoder --- .../cartodb_services/google/geocoder.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 2fb0e01..a741539 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -8,9 +8,17 @@ from exceptions import MalformedResult from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory +from multiprocessing import Pool, TimeoutError + +import json + +def async_geocoder(geocoder, address): + results = geocoder.geocode(address=address) + return results if results else [] class GoogleMapsGeocoder: """A Google Maps Geocoder wrapper for python""" + PARALLEL_PROCESSES = 13 def __init__(self, client_id, client_secret, logger): if client_id is None: @@ -33,6 +41,45 @@ class GoogleMapsGeocoder: except KeyError: raise MalformedResult() + def bulk_geocode(self, searchtext): + try: + decoded_searchtext = json.loads(searchtext) + except Exception as e: + self._logger.error('General error', exception=e) + raise e + + bulk_results = {} + pool = Pool(processes=self.PARALLEL_PROCESSES) + for search in decoded_searchtext: + search_id, address = [search[k] for k in ['id', 'address']] + if address: + result = pool.apply_async(async_geocoder, + (self.geocoder, address)) + else: + result = [] + bulk_results[search_id] = result + pool.close() + pool.join() + + try: + results = [] + for search_id, bulk_result in bulk_results.items(): + try: + result = bulk_result.get() + except Exception as e: + self._logger.error('Error at Google async_geocoder', e) + result = [] + + lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] + results.append([search_id, lng_lat, []]) + return results + except KeyError as e: + self._logger.error('KeyError error', exception=e) + raise MalformedResult() + except Exception as e: + self._logger.error('General error', exception=e) + raise e + def _extract_lng_lat_from_result(self, result): location = result['geometry']['location'] longitude = location['lng'] From 34fc6439d2f646d0cde6b603f051d5efcc8e607d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 11 Jun 2018 16:12:29 +0200 Subject: [PATCH 060/265] cdb_bulk_geocode_street_point functions --- client/cdb_dataservices_client--0.24.0.sql | 78 +++++++++++++++++++ client/renderer/interface.yaml | 7 ++ client/sql/16_custom_types.sql | 6 ++ .../cdb_dataservices_server--0.31.0.sql | 7 ++ .../extension/sql/21_bulk_geocode_street.sql | 75 ++++++++++++++++++ 5 files changed, 173 insertions(+) create mode 100644 server/extension/sql/21_bulk_geocode_street.sql diff --git a/client/cdb_dataservices_client--0.24.0.sql b/client/cdb_dataservices_client--0.24.0.sql index 14a3af6..48bfb15 100644 --- a/client/cdb_dataservices_client--0.24.0.sql +++ b/client/cdb_dataservices_client--0.24.0.sql @@ -63,6 +63,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( the_geom geometry(Multipolygon,4326) ); +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE TYPE cdb_dataservices_client.simple_route AS ( shape geometry(LineString,4326), length real, @@ -400,6 +406,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -2377,6 +2408,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -4301,6 +4368,14 @@ RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) @@ -4809,6 +4884,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 874e99a..71a7358 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -70,6 +70,13 @@ - { name: state_province, type: text, default: 'NULL'} - { name: country, type: text, default: 'NULL'} +- name: cdb_bulk_geocode_street_point + return_type: SETOF cdb_dataservices_client.geocoding + multi_row: true + multi_field: true + params: + - { name: searchtext, type: jsonb } + - name: cdb_here_geocode_street_point return_type: Geometry params: diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 2b77cdb..1b74cac 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -4,6 +4,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( the_geom geometry(Multipolygon,4326) ); +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE TYPE cdb_dataservices_client.simple_route AS ( shape geometry(LineString,4326), length real, diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql index 2e7b27c..c4360c4 100644 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -2341,6 +2341,13 @@ RETURNS VOID AS $$ config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql new file mode 100644 index 0000000..994594e --- /dev/null +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -0,0 +1,75 @@ +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + result = plpy.execute(google_plan, [username, orgname, searchtext]) + return result + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + + logger = Logger(logger_config) + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + geocode_results = geocoder.bulk_geocode(searchtext=searchtext) + if geocode_results: + results = [] + for result in geocode_results: + if result[1]: + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) + point = plpy.execute(plan, result[1], 1)[0] + results.append([result[0], point['the_geom'], None]) + else: + results.append([result[0], None, None]) + service_manager.quota_service.increment_success_service_use(len(results)) + return results + else: + service_manager.quota_service.increment_empty_service_use(len(searchtext)) + return [] + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use(len(searchtext)) + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to bulk geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + From f3f2b213e72f9b38bcc5399e7ae85c1e2e9d4998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 11 Jun 2018 16:25:09 +0200 Subject: [PATCH 061/265] Bump versions --- client/cdb_dataservices_client--0.24.0.sql | 78 - client/cdb_dataservices_client--0.25.0.sql | 5038 +++++++++++++++++ client/cdb_dataservices_client.control | 2 +- .../cdb_dataservices_server--0.31.0.sql | 7 - .../cdb_dataservices_server--0.32.0.sql | 3733 ++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- server/lib/python/cartodb_services/setup.py | 2 +- 7 files changed, 8774 insertions(+), 88 deletions(-) create mode 100644 client/cdb_dataservices_client--0.25.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.32.0.sql diff --git a/client/cdb_dataservices_client--0.24.0.sql b/client/cdb_dataservices_client--0.24.0.sql index 48bfb15..14a3af6 100644 --- a/client/cdb_dataservices_client--0.24.0.sql +++ b/client/cdb_dataservices_client--0.24.0.sql @@ -63,12 +63,6 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( the_geom geometry(Multipolygon,4326) ); -CREATE TYPE cdb_dataservices_client.geocoding AS ( - cartodb_id integer, - the_geom geometry(Multipolygon,4326), - metadata jsonb -); - CREATE TYPE cdb_dataservices_client.simple_route AS ( shape geometry(LineString,4326), length real, @@ -406,31 +400,6 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) -RETURNS SETOF cdb_dataservices_client.geocoding AS $$ -DECLARE - - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; --- --- Public dataservices API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -2408,42 +2377,6 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- -CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) -RETURNS SETOF cdb_dataservices_client.geocoding AS $$ -DECLARE - - username text; - orgname text; - _returned_sqlstate TEXT; - _message_text TEXT; - _pg_exception_context TEXT; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - - - BEGIN - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); - EXCEPTION - WHEN OTHERS THEN - GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, - _message_text = MESSAGE_TEXT, - _pg_exception_context = PG_EXCEPTION_CONTEXT; - RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; - - END; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; --- --- Exception-safe private DataServices API function --- - CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -4368,14 +4301,6 @@ RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); -$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); -CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) -RETURNS SETOF cdb_dataservices_client.geocoding AS $$ - CONNECT cdb_dataservices_client._server_conn_str(); - - SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext); - $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) @@ -4884,9 +4809,6 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; - GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql new file mode 100644 index 0000000..48bfb15 --- /dev/null +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -0,0 +1,5038 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- For the OBS_Meta functions +CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params) + ) + ) + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + # Create a new table with the required columns + plpy.execute('CREATE TABLE "{schema}".{table_name} ( ' + 'cartodb_id int, the_geom geometry, {columns_with_types} ' + ');' + .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute( + "SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params))) + + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [ + colnames_arr[i] + + ' ' + + coltypes_arr[i] for i in range( + 0, + len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + aliased_colname_list = ','.join( + ['result.' + name for name in colnames_arr]) + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute( + "SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, " + "{schema}::text, {dbname}::text, {table_name}::text);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + user_db_role=plpy.quote_literal(user_db_role), + schema=plpy.quote_literal(user_schema), + dbname=plpy.quote_literal(dbname), + table_name=plpy.quote_literal(table_name))) + + if ds_fdw_metadata[0]["schemaname"]: + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + else: + raise Exception('Error connecting dataset via FDW') + + # Create a new table with the required columns + plpy.execute( + 'INSERT INTO "{schema}".{analysis_table_name} ' + 'SELECT ut.cartodb_id, ut.the_geom, {colname_list} ' + 'FROM "{schema}".{table_name} ut ' + 'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, ' + '{function_name}::text, {params}::json) ' + 'AS result ({columns_with_types}, cartodb_id int) ' + 'ON result.cartodb_id = ut.cartodb_id;' .format( + schema=user_schema, + analysis_table_name=output_table_name, + colname_list=aliased_colname_list, + table_name=table_name, + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params), + columns_with_types=columns_with_types)) + + # Wipe user FDW data from the server + wiped = plpy.execute( + "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, " + "{server_table_name}::text, {fdw_server}::text)" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + fdw_server=plpy.quote_literal(server_name))) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text +)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata( + username text, + orgname text, + function_name text, + params json +) RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData( + username text, + orgname text, + table_schema text, + table_name text, + function_name text, + params json +) RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( + username text, + orgname text, + table_schema text, + table_name text, + server_name text +) RETURNS boolean AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, orgname, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, orgname, ip_address); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, orgname, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_with_waypoints (username, orgname, waypoints, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundary (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundaryid (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundarybyid (username, orgname, geometry_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasure (username, orgname, geom, measure_id, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasurebyid (username, orgname, geom_ref, measure_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomvals, params, merge); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json) +RETURNS TABLE(id text, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomrefs, params); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeta (username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_metadatavalidation (username, orgname, geom_extent, geom_type, params, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getcategory (username, orgname, geom, category_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, orgname, geom, name, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensuscategory (username, orgname, geom, name, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getpopulation (username, orgname, geom, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_search (username, orgname, search_term, relevant_boundary); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailableboundaries (username, orgname, geom, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_dumpversion (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablenumerators (username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, section_tags text[] DEFAULT ARRAY[]::TEXT[], subsection_tags text[] DEFAULT ARRAY[]::TEXT[], other_tags text[] DEFAULT ARRAY[]::TEXT[], ids text[] DEFAULT ARRAY[]::TEXT[], name text DEFAULT NULL, denom_id text DEFAULT '', geom_id text DEFAULT '', timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._obs_getnumerators (username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabledenominators (username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, timespan text DEFAULT NULL, number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablegeometries (username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabletimespans (username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_legacybuildermetadata (username, orgname, aggregate_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info (username text, orgname text) +RETURNS SETOF service_quota_info AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) +RETURNS BOOLEAN AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_enough_quota (username, orgname, service, input_size); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck( + source_query text, + parameters json +) RETURNS boolean AS $$ +DECLARE + errors text[]; + validator_errors text[]; + validator_error text; + valid boolean; + geoms record; +BEGIN + errors := (ARRAY[])::TEXT[]; + FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type, + bool_and(st_isvalid(the_geom)) as valid, + avg(st_npoints(the_geom)) as avg_vertex, + ST_SetSRID(ST_Extent(the_geom), 4326) as extent, + count(*)::INT as numgeoms + FROM (%s) as _source + GROUP BY ST_GeometryType(the_geom)', source_query) + LOOP + IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN + errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type)); + END IF; + + IF geoms.valid IS FALSE THEN + errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them'); + END IF; + + -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here + -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813 + IF geoms.avg_vertex > 1000 THEN + errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them'); + END IF; + + -- OBS specific part + EXECUTE 'SELECT valid, errors + FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)' + INTO valid, validator_errors + USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms; + IF valid is FALSE THEN + FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors + LOOP + errors := array_append(errors, validator_error); + END LOOP; + END IF; + END LOOP; + + IF CARDINALITY(errors) > 0 THEN + RAISE EXCEPTION '%', errors; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_with_waypoints(waypoints geometry(Point, 4326)[], mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom Geometry, category_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom Geometry, name text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom Geometry, name text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text ) TO publicuser; + + + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index b56df21..3037739 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.24.0' +default_version = '0.25.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql index c4360c4..2e7b27c 100644 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -2341,13 +2341,6 @@ RETURNS VOID AS $$ config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; --- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? -CREATE TYPE cdb_dataservices_server.geocoding AS ( - cartodb_id integer, - the_geom geometry(Multipolygon,4326), - metadata jsonb -); - CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql new file mode 100644 index 0000000..ed2cdd7 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -0,0 +1,3733 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + result = plpy.execute(google_plan, [username, orgname, searchtext]) + return result + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + + logger = Logger(logger_config) + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + geocode_results = geocoder.bulk_geocode(searchtext=searchtext) + if geocode_results: + results = [] + for result in geocode_results: + if result[1]: + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) + point = plpy.execute(plan, result[1], 1)[0] + results.append([result[0], point['the_geom'], None]) + else: + results.append([result[0], None, None]) + service_manager.quota_service.increment_success_service_use(len(results)) + return results + else: + service_manager.quota_service.increment_empty_service_use(len(searchtext)) + return [] + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use(len(searchtext)) + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to bulk geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index fe7e040..908a91d 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.31.0' +default_version = '0.32.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 61c8a3c..3ea820e 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.18.0', + version='0.19.0', description='CartoDB Services API Python Library', From e85f43f1d10f0f1257ba587c542b99b8d233dbd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 15 Jun 2018 09:55:52 +0200 Subject: [PATCH 062/265] cdb_bulk_geocode_street_point skeleton --- client/cdb_dataservices_client--0.25.0.sql | 30 +++++++++++++------ client/renderer/interface.yaml | 2 +- client/sql/21_bulk_geocoding_functions.sql | 12 ++++++++ .../cdb_dataservices_server--0.32.0.sql | 2 +- .../extension/sql/21_bulk_geocode_street.sql | 2 +- 5 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 client/sql/21_bulk_geocoding_functions.sql diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 48bfb15..536bc10 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -406,7 +406,7 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searchtext jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -422,7 +422,7 @@ BEGIN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searchtext); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- @@ -1972,6 +1972,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( CONNECT cdb_dataservices_client._server_conn_str(); TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + -- TODO: check + -- TODO: bulk + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- -- Exception-safe private DataServices API function -- @@ -2408,7 +2420,7 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- -CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -2429,7 +2441,7 @@ BEGIN BEGIN - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searchtext); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, @@ -4369,12 +4381,12 @@ RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); -CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ CONNECT cdb_dataservices_client._server_conn_str(); - SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext); + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searchtext); $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); @@ -4884,8 +4896,8 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 71a7358..22c3005 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -70,7 +70,7 @@ - { name: state_province, type: text, default: 'NULL'} - { name: country, type: text, default: 'NULL'} -- name: cdb_bulk_geocode_street_point +- name: _cdb_bulk_geocode_street_point return_type: SETOF cdb_dataservices_client.geocoding multi_row: true multi_field: true diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql new file mode 100644 index 0000000..2a1f259 --- /dev/null +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -0,0 +1,12 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + -- TODO: check + -- TODO: bulk + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index ed2cdd7..59239e9 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2348,7 +2348,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 994594e..377b825 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -5,7 +5,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger From 58d70e252f24f8675697284ed95a2c75e78f1e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 20 Jun 2018 18:30:03 +0200 Subject: [PATCH 063/265] checked, bulked cdb_bulk_geocode_street_point --- .gitignore | 2 + client/cdb_dataservices_client--0.25.0.sql | 74 +++++++++++++++++-- client/sql/05_utils.sql | 15 ++++ client/sql/21_bulk_geocoding_functions.sql | 58 +++++++++++++-- .../cartodb_services/google/geocoder.py | 5 ++ 5 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 client/sql/05_utils.sql diff --git a/.gitignore b/.gitignore index 3cf5f35..ee5b680 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ cartodb_services.egg-info/ build/ dist/ .vscode/ +.idea/ +venv/ diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 536bc10..088f949 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -4,7 +4,21 @@ -- Make sure we have a sane search path to create/update the extension SET search_path = "$user",cartodb,public,cdb_dataservices_client; --- +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql;-- -- Geocoder server connection config -- -- The purpose of this function is provide to the PL/Proxy functions @@ -1972,18 +1986,62 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( CONNECT cdb_dataservices_client._server_conn_str(); TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE + monthly_quota integer; + used_quota integer; + soft_limit boolean; + provider text; - username text; - orgname text; + remaining_quota integer; + estimated_row_count integer; + + cartodb_id_batch integer; + batches_n integer; + BATCHES_SIZE CONSTANT numeric := 100; + current_row_count integer ; BEGIN - -- TODO: check - -- TODO: bulk - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext); + SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider + into monthly_quota, used_quota, soft_limit, provider + FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + + remaining_quota := monthly_quota - used_quota; + estimated_row_count := count_estimate(query); + RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', + estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; + IF estimated_row_count > remaining_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + END IF; + + EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; + + RAISE DEBUG 'batches_n: %', batches_n; + + CREATE TEMPORARY TABLE bulk_geocode_street_point + (cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb); + + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT json_build_object(''id'', cartodb_id, ''address'', %s) as data , floor((cartodb_id-1)::float/$1) as batch' || + ' FROM (%s) _x' || + ')' || + 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', searchtext, query) + USING BATCHES_SIZE, cartodb_id_batch; + + GET DIAGNOSTICS current_row_count = ROW_COUNT; + RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; + + END LOOP; + + RETURN QUERY SELECT * FROM bulk_geocode_street_point; END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; -- -- Exception-safe private DataServices API function -- diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql new file mode 100644 index 0000000..d161c6a --- /dev/null +++ b/client/sql/05_utils.sql @@ -0,0 +1,15 @@ +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 2a1f259..12766fc 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,12 +1,56 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE + monthly_quota integer; + used_quota integer; + soft_limit boolean; + provider text; - username text; - orgname text; + remaining_quota integer; + estimated_row_count integer; + + cartodb_id_batch integer; + batches_n integer; + BATCHES_SIZE CONSTANT numeric := 100; + current_row_count integer ; BEGIN - -- TODO: check - -- TODO: bulk - RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext); + SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider + into monthly_quota, used_quota, soft_limit, provider + FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + + remaining_quota := monthly_quota - used_quota; + estimated_row_count := count_estimate(query); + RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', + estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; + IF estimated_row_count > remaining_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + END IF; + + EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; + + RAISE DEBUG 'batches_n: %', batches_n; + + CREATE TEMPORARY TABLE bulk_geocode_street_point + (cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb); + + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT json_build_object(''id'', cartodb_id, ''address'', %s) as data , floor((cartodb_id-1)::float/$1) as batch' || + ' FROM (%s) _x' || + ')' || + 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', searchtext, query) + USING BATCHES_SIZE, cartodb_id_batch; + + GET DIAGNOSTICS current_row_count = ROW_COUNT; + RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; + + END LOOP; + + RETURN QUERY SELECT * FROM bulk_geocode_street_point; END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index a741539..3dff728 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -12,7 +12,12 @@ from multiprocessing import Pool, TimeoutError import json +import time, random + def async_geocoder(geocoder, address): + # time.sleep(.3 + random.random()) + # return [{ 'geometry': { 'location': { 'lng': 1, 'lat': 2 } } }] + results = geocoder.geocode(address=address) return results if results else [] From 4d2abc7667bf62ba9c8963fce2609266691ae61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 22 Jun 2018 09:41:29 +0200 Subject: [PATCH 064/265] Simpler and more precise quota check --- client/cdb_dataservices_client--0.25.0.sql | 24 ++++++++-------------- client/sql/21_bulk_geocoding_functions.sql | 24 ++++++++-------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 088f949..6d87658 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1989,29 +1989,21 @@ $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE - monthly_quota integer; - used_quota integer; - soft_limit boolean; - provider text; - - remaining_quota integer; - estimated_row_count integer; + query_row_count integer; + enough_quota boolean; cartodb_id_batch integer; batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; BEGIN - SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider - into monthly_quota, used_quota, soft_limit, provider - FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - remaining_quota := monthly_quota - used_quota; - estimated_row_count := count_estimate(query); - RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', - estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; - IF estimated_row_count > remaining_quota THEN - RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', + query_row_count, query, searchtext; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NOT NULL AND enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 12766fc..093cb60 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,29 +1,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE - monthly_quota integer; - used_quota integer; - soft_limit boolean; - provider text; - - remaining_quota integer; - estimated_row_count integer; + query_row_count integer; + enough_quota boolean; cartodb_id_batch integer; batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; BEGIN - SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider - into monthly_quota, used_quota, soft_limit, provider - FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - remaining_quota := monthly_quota - used_quota; - estimated_row_count := count_estimate(query); - RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', - estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; - IF estimated_row_count > remaining_quota THEN - RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', + query_row_count, query, searchtext; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NOT NULL AND enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; From bbbf70f3ace5a121a7e8669e5f716c8d0014da1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 22 Jun 2018 12:19:51 +0200 Subject: [PATCH 065/265] Street level columns for country, city and state --- client/cdb_dataservices_client--0.25.0.sql | 13 ++++++++----- client/sql/21_bulk_geocoding_functions.sql | 13 ++++++++----- .../cdb_dataservices_server--0.32.0.sql | 8 ++++---- .../extension/sql/21_bulk_geocode_street.sql | 8 ++++---- .../cartodb_services/google/geocoder.py | 19 ++++++++++++------- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 6d87658..56f8826 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1986,7 +1986,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( CONNECT cdb_dataservices_client._server_conn_str(); TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + country_column text, state_column text, city_column text, street_column text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; @@ -1999,8 +2000,8 @@ DECLARE BEGIN EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', - query_row_count, query, searchtext; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; IF enough_quota IS NOT NULL AND enough_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; @@ -2018,12 +2019,14 @@ BEGIN EXECUTE format( 'WITH geocoding_data as (' || - ' SELECT json_build_object(''id'', cartodb_id, ''address'', %s) as data , floor((cartodb_id-1)::float/$1) as batch' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((cartodb_id-1)::float/$1) as batch' || ' FROM (%s) _x' || ')' || 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', searchtext, query) + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query) USING BATCHES_SIZE, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 093cb60..c103100 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,4 +1,5 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + country_column text, state_column text, city_column text, street_column text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; @@ -11,8 +12,8 @@ DECLARE BEGIN EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', - query_row_count, query, searchtext; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; IF enough_quota IS NOT NULL AND enough_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; @@ -30,12 +31,14 @@ BEGIN EXECUTE format( 'WITH geocoding_data as (' || - ' SELECT json_build_object(''id'', cartodb_id, ''address'', %s) as data , floor((cartodb_id-1)::float/$1) as batch' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((cartodb_id-1)::float/$1) as batch' || ' FROM (%s) _x' || ')' || 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', searchtext, query) + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query) USING BATCHES_SIZE, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 59239e9..775c904 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2375,7 +2375,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger from cartodb_services.google import GoogleMapsGeocoder @@ -2389,7 +2389,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ try: service_manager.assert_within_limits(quota=False) geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) - geocode_results = geocoder.bulk_geocode(searchtext=searchtext) + geocode_results = geocoder.bulk_geocode(searches=searches) if geocode_results: results = [] for result in geocode_results: @@ -2402,10 +2402,10 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager.quota_service.increment_success_service_use(len(results)) return results else: - service_manager.quota_service.increment_empty_service_use(len(searchtext)) + service_manager.quota_service.increment_empty_service_use(len(searches)) return [] except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use(len(searchtext)) + service_manager.quota_service.increment_failed_service_use(len(searches)) return [] except BaseException as e: import sys diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 377b825..aebb6b8 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -32,7 +32,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger from cartodb_services.google import GoogleMapsGeocoder @@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ try: service_manager.assert_within_limits(quota=False) geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) - geocode_results = geocoder.bulk_geocode(searchtext=searchtext) + geocode_results = geocoder.bulk_geocode(searches=searches) if geocode_results: results = [] for result in geocode_results: @@ -59,10 +59,10 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ service_manager.quota_service.increment_success_service_use(len(results)) return results else: - service_manager.quota_service.increment_empty_service_use(len(searchtext)) + service_manager.quota_service.increment_empty_service_use(len(searches)) return [] except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use(len(searchtext)) + service_manager.quota_service.increment_failed_service_use(len(searches)) return [] except BaseException as e: import sys diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 3dff728..ba0a663 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -14,16 +14,18 @@ import json import time, random -def async_geocoder(geocoder, address): +def async_geocoder(geocoder, address, components): + # TODO: clean this and previous import # time.sleep(.3 + random.random()) # return [{ 'geometry': { 'location': { 'lng': 1, 'lat': 2 } } }] - results = geocoder.geocode(address=address) + results = geocoder.geocode(address=address, components=components) return results if results else [] class GoogleMapsGeocoder: """A Google Maps Geocoder wrapper for python""" PARALLEL_PROCESSES = 13 + SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] def __init__(self, client_id, client_secret, logger): if client_id is None: @@ -46,20 +48,23 @@ class GoogleMapsGeocoder: except KeyError: raise MalformedResult() - def bulk_geocode(self, searchtext): + def bulk_geocode(self, searches): try: - decoded_searchtext = json.loads(searchtext) + decoded_searches = json.loads(searches) except Exception as e: self._logger.error('General error', exception=e) raise e bulk_results = {} pool = Pool(processes=self.PARALLEL_PROCESSES) - for search in decoded_searchtext: - search_id, address = [search[k] for k in ['id', 'address']] + for search in decoded_searches: + search_id, address, city, state, country = \ + [search.get(k, None) for k in self.SEARCH_KEYS] + opt_params = self._build_optional_parameters(city, state, country) if address: + self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) result = pool.apply_async(async_geocoder, - (self.geocoder, address)) + (self.geocoder, address, opt_params)) else: result = [] bulk_results[search_id] = result From 91d93bef792302c31127605ee1fabad1211e0081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 22 Jun 2018 12:35:29 +0200 Subject: [PATCH 066/265] Random temporary table name --- client/cdb_dataservices_client--0.25.0.sql | 17 +++++++++++------ client/sql/21_bulk_geocoding_functions.sql | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 56f8826..4fa6f2b 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1997,6 +1997,8 @@ DECLARE batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; + + temp_table_name text; BEGIN EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; @@ -2011,8 +2013,11 @@ BEGIN RAISE DEBUG 'batches_n: %', batches_n; - CREATE TEMPORARY TABLE bulk_geocode_street_point - (cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb); + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); FOR cartodb_id_batch in 0..(batches_n - 1) LOOP @@ -2023,10 +2028,10 @@ BEGIN ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || ' floor((cartodb_id-1)::float/$1) as batch' || ' FROM (%s) _x' || - ')' || - 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query) + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING BATCHES_SIZE, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; @@ -2034,7 +2039,7 @@ BEGIN END LOOP; - RETURN QUERY SELECT * FROM bulk_geocode_street_point; + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; -- diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index c103100..cc15023 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -9,6 +9,8 @@ DECLARE batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; + + temp_table_name text; BEGIN EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; @@ -23,8 +25,11 @@ BEGIN RAISE DEBUG 'batches_n: %', batches_n; - CREATE TEMPORARY TABLE bulk_geocode_street_point - (cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb); + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); FOR cartodb_id_batch in 0..(batches_n - 1) LOOP @@ -35,10 +40,10 @@ BEGIN ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || ' floor((cartodb_id-1)::float/$1) as batch' || ' FROM (%s) _x' || - ')' || - 'INSERT INTO bulk_geocode_street_point SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query) + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING BATCHES_SIZE, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; @@ -46,6 +51,6 @@ BEGIN END LOOP; - RETURN QUERY SELECT * FROM bulk_geocode_street_point; + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; From 9b64d9199816adaa6ade7f9d3249ef36ef01a65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 26 Jun 2018 10:56:14 +0200 Subject: [PATCH 067/265] Fixed env variables cleaning --- test/run_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/run_tests.py b/test/run_tests.py index 61dc3ef..7308010 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -74,11 +74,11 @@ def set_environment_variables(username, api_key, table_name, host, schema): def clean_environment_variables(): - del os.environ["GEOCODER_API_TEST_USERNAME"] - del os.environ["GEOCODER_API_TEST_API_KEY"] - del os.environ["GEOCODER_API_TEST_TABLE_NAME"] - del os.environ["GEOCODER_API_TEST_HOST"] - del os.environ["GEOCODER_API_TEST_SCHEMA"] + os.environ.pop("GEOCODER_API_TEST_USERNAME", None) + os.environ.pop("GEOCODER_API_TEST_API_KEY", None) + os.environ.pop("GEOCODER_API_TEST_TABLE_NAME", None) + os.environ.pop("GEOCODER_API_TEST_HOST", None) + os.environ.pop("GEOCODER_API_TEST_SCHEMA", None) if __name__ == "__main__": main() From e2804444799f264487bf0467e7586ce5df72722d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 26 Jun 2018 10:56:39 +0200 Subject: [PATCH 068/265] Integration tests dependencies --- test/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 test/requirements.txt diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..e7027e2 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,3 @@ +# Integration tests dependencies +requests==2.9.1 +nose==1.3.7 From c2a207b1cdab21090d69592dd4aa21aa5419f323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 26 Jun 2018 13:35:24 +0200 Subject: [PATCH 069/265] Batching, better support for null columns, and bulk geocoding integration tests --- client/cdb_dataservices_client--0.25.0.sql | 20 +++- client/sql/21_bulk_geocoding_functions.sql | 20 +++- test/helpers/integration_test_helper.py | 8 +- test/integration/test_street_functions.py | 101 ++++++++++++++++++++- 4 files changed, 133 insertions(+), 16 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 4fa6f2b..f1346e9 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1987,7 +1987,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - country_column text, state_column text, city_column text, street_column text) + country_column text, state_column text, city_column text, street_column text, batch_size integer DEFAULT 100) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; @@ -1995,11 +1995,18 @@ DECLARE cartodb_id_batch integer; batches_n integer; - BATCHES_SIZE CONSTANT numeric := 100; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_BATCH_SIZE CONSTANT numeric := 1000; current_row_count integer ; temp_table_name text; BEGIN + IF batch_size IS NULL THEN + batch_size := DEFAULT_BATCH_SIZE; + ELSIF batch_size > MAX_BATCH_SIZE THEN + RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; + END IF; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', @@ -2009,7 +2016,7 @@ BEGIN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; - EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; + EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', batch_size, query) INTO batches_n; RAISE DEBUG 'batches_n: %', batches_n; @@ -2019,6 +2026,11 @@ BEGIN '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', temp_table_name); + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + FOR cartodb_id_batch in 0..(batches_n - 1) LOOP @@ -2032,7 +2044,7 @@ BEGIN 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING BATCHES_SIZE, cartodb_id_batch; + USING batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index cc15023..a733245 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - country_column text, state_column text, city_column text, street_column text) + country_column text, state_column text, city_column text, street_column text, batch_size integer DEFAULT 100) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; @@ -7,11 +7,18 @@ DECLARE cartodb_id_batch integer; batches_n integer; - BATCHES_SIZE CONSTANT numeric := 100; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_BATCH_SIZE CONSTANT numeric := 1000; current_row_count integer ; temp_table_name text; BEGIN + IF batch_size IS NULL THEN + batch_size := DEFAULT_BATCH_SIZE; + ELSIF batch_size > MAX_BATCH_SIZE THEN + RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; + END IF; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', @@ -21,7 +28,7 @@ BEGIN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; - EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; + EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', batch_size, query) INTO batches_n; RAISE DEBUG 'batches_n: %', batches_n; @@ -31,6 +38,11 @@ BEGIN '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', temp_table_name); + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + FOR cartodb_id_batch in 0..(batches_n - 1) LOOP @@ -44,7 +56,7 @@ BEGIN 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING BATCHES_SIZE, cartodb_id_batch; + USING batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index 1c51533..7fdcd38 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -22,13 +22,15 @@ class IntegrationTestHelper: } @classmethod - def execute_query(cls, sql_api_url, query): + def execute_query_raw(cls, sql_api_url, query): requests.packages.urllib3.disable_warnings() query_url = "{0}?q={1}".format(sql_api_url, query) print "Executing query: {0}".format(query_url) query_response = requests.get(query_url) if query_response.status_code != 200: raise Exception(json.loads(query_response.text)['error']) - query_response_data = json.loads(query_response.text) + return json.loads(query_response.text) - return query_response_data['rows'][0] + @classmethod + def execute_query(cls, sql_api_url, query): + return cls.execute_query_raw(sql_api_url, query)['rows'][0] diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index a409864..180cc70 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -1,10 +1,13 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + from unittest import TestCase from nose.tools import assert_raises from nose.tools import assert_not_equal, assert_equal from ..helpers.integration_test_helper import IntegrationTestHelper -class TestStreetFunctions(TestCase): +class TestStreetFunctionsSetUp(TestCase): def setUp(self): self.env_variables = IntegrationTestHelper.get_environment_variables() @@ -15,19 +18,107 @@ class TestStreetFunctions(TestCase): self.env_variables['api_key'] ) + +class TestStreetFunctions(TestStreetFunctionsSetUp): + def test_if_select_with_street_point_is_ok(self): query = "SELECT cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_street_without_api_key_raise_error(self): query = "SELECT cdb_geocode_street_point(street) " \ - "as geometry FROM {0} LIMIT 1".format( - self.env_variables['table_name']) + "as geometry FROM {0} LIMIT 1".format( + self.env_variables['table_name']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: assert_equal(e.message[0], "The api_key must be provided") + + +class TestBulkStreetFunctions(TestStreetFunctionsSetUp): + + def test_full_spec(self): + query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 1 as cartodb_id, ''Spain'' as country, " \ + "''Castilla y León'' as state, ''Valladolid'' as city, " \ + "''Plaza Mayor'' as street " \ + "UNION " \ + "select 2 as cartodb_id, ''Spain'' as country, " \ + "''Castilla y León'' as state, ''Valladolid'' as city, " \ + "''Paseo Zorrilla'' as street' " \ + ", 'country', 'state', 'city', 'street')" + response = self._run_authenticated(query) + + assert_equal(response['total_rows'], 2) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -3.7074009, 40.415511) + self._assert_x_y(row_by_cartodb_id[2], -4.7404453, 41.6314339) + + def test_empty_columns(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "'''''', '''''', '''''', 'address')" + response = self._run_authenticated(query) + + assert_equal(response['total_rows'], 1) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -122.0885504, 37.4238657) + + def test_null_columns(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "null, null, null, 'address')" + response = self._run_authenticated(query) + + assert_equal(response['total_rows'], 1) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -122.0885504, 37.4238657) + + def test_batching(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1900 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 2, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 3, \"address\": \"1902 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "null, null, null, 'address', 2)" + response = self._run_authenticated(query) + # from nose.tools import set_trace; set_trace() + + assert_equal(response['total_rows'], 3) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -122.0875324, 37.4227968) + self._assert_x_y(row_by_cartodb_id[2], -122.0885504, 37.4238657) + self._assert_x_y(row_by_cartodb_id[3], -122.0876674, 37.4235729) + + + def _run_authenticated(self, query): + authenticated_query = "{}&api_key={}".format(query, + self.env_variables[ + 'api_key']) + return IntegrationTestHelper.execute_query_raw(self.sql_api_url, + authenticated_query) + + def _row_by_cartodb_id(self, response): + return {r['cartodb_id']: r for r in response['rows']} + + def _assert_x_y(self, row, expected_x, expected_y): + assert_equal(row['st_x'], expected_x) + assert_equal(row['st_y'], expected_y) + From d5e47e39ab76ca77765466e4b9858526d4ae3e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 26 Jun 2018 13:59:44 +0200 Subject: [PATCH 070/265] Revert bulk geocoding column parameters order and test_city_column_geocoding --- client/cdb_dataservices_client--0.25.0.sql | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- test/integration/test_street_functions.py | 27 ++++++++++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index f1346e9..6900c64 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1987,7 +1987,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - country_column text, state_column text, city_column text, street_column text, batch_size integer DEFAULT 100) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 100) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index a733245..b3ff689 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - country_column text, state_column text, city_column text, street_column text, batch_size integer DEFAULT 100) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 100) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 180cc70..3694468 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -51,7 +51,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "select 2 as cartodb_id, ''Spain'' as country, " \ "''Castilla y León'' as state, ''Valladolid'' as city, " \ "''Paseo Zorrilla'' as street' " \ - ", 'country', 'state', 'city', 'street')" + ", 'street', 'city', 'state', 'country')" response = self._run_authenticated(query) assert_equal(response['total_rows'], 2) @@ -66,7 +66,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'select * from jsonb_to_recordset(''[" \ "{\"cartodb_id\": 1, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ - "'''''', '''''', '''''', 'address')" + "'address', '''''', '''''', '''''')" response = self._run_authenticated(query) assert_equal(response['total_rows'], 1) @@ -80,7 +80,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'select * from jsonb_to_recordset(''[" \ "{\"cartodb_id\": 1, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ - "null, null, null, 'address')" + "'address')" response = self._run_authenticated(query) assert_equal(response['total_rows'], 1) @@ -96,9 +96,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "{\"cartodb_id\": 2, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}," \ "{\"cartodb_id\": 3, \"address\": \"1902 amphitheatre parkway, mountain view, ca, us\"}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ - "null, null, null, 'address', 2)" + "'address', null, null, null, 2)" response = self._run_authenticated(query) - # from nose.tools import set_trace; set_trace() assert_equal(response['total_rows'], 3) @@ -108,6 +107,24 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self._assert_x_y(row_by_cartodb_id[3], -122.0876674, 37.4235729) + def test_city_column_geocoding(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"city\": \"Valladolid\"}," \ + "{\"cartodb_id\": 2, \"city\": \"Madrid\"}" \ + "]''::jsonb) as (cartodb_id integer, city text)', " \ + "'city')" + response = self._run_authenticated(query) + # from nose.tools import set_trace; set_trace() + + assert_equal(response['total_rows'], 2) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -4.7245321, 41.652251) + self._assert_x_y(row_by_cartodb_id[2], -3.7037902, 40.4167754) + + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From c13d29e4c2b3a5ccc0ad44075b67ff5aca75c11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 26 Jun 2018 16:52:41 +0200 Subject: [PATCH 071/265] test_free_text_geocoding --- test/integration/test_street_functions.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 3694468..9214c2f 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -106,7 +106,6 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self._assert_x_y(row_by_cartodb_id[2], -122.0885504, 37.4238657) self._assert_x_y(row_by_cartodb_id[3], -122.0876674, 37.4235729) - def test_city_column_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ @@ -116,7 +115,6 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "]''::jsonb) as (cartodb_id integer, city text)', " \ "'city')" response = self._run_authenticated(query) - # from nose.tools import set_trace; set_trace() assert_equal(response['total_rows'], 2) @@ -124,6 +122,22 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self._assert_x_y(row_by_cartodb_id[1], -4.7245321, 41.652251) self._assert_x_y(row_by_cartodb_id[2], -3.7037902, 40.4167754) + def test_free_text_geocoding(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from (" \ + "select 1 as cartodb_id, ''W 26th Street'' as address, " \ + "null as city , null as state , null as country" \ + ")_x', " \ + "'''Logroño, La Rioja, Spain''')" + response = self._run_authenticated(query) + # from nose.tools import set_trace; set_trace() + + assert_equal(response['total_rows'], 1) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From 675ef72e30462c769d801837bd471a7186de7582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 08:51:40 +0200 Subject: [PATCH 072/265] test_templating_geocoding --- test/integration/test_street_functions.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 9214c2f..1ce39b8 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -131,13 +131,31 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): ")_x', " \ "'''Logroño, La Rioja, Spain''')" response = self._run_authenticated(query) - # from nose.tools import set_trace; set_trace() assert_equal(response['total_rows'], 1) row_by_cartodb_id = self._row_by_cartodb_id(response) self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) + def test_templating_geocoding(self): + query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ + "cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 1 as cartodb_id, ''Logroño'' as city', " \ + "'city || '', '' || ''Spain''') " \ + "UNION " \ + "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ + "cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 2 as cartodb_id, ''Logroño'' as city', " \ + "'city || '', '' || ''Argentina''')" + response = self._run_authenticated(query) + # from nose.tools import set_trace; set_trace() + + assert_equal(response['total_rows'], 2) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) + self._assert_x_y(row_by_cartodb_id[2], -61.6961807, -29.5031057) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From ed828c3b89497a7c09c9d49d67908f9e80c4d70f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 10:13:59 +0200 Subject: [PATCH 073/265] test_templating_with_two_columns_geocoding --- test/integration/test_street_functions.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 1ce39b8..5f8ce60 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -148,7 +148,6 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'select 2 as cartodb_id, ''Logroño'' as city', " \ "'city || '', '' || ''Argentina''')" response = self._run_authenticated(query) - # from nose.tools import set_trace; set_trace() assert_equal(response['total_rows'], 2) @@ -156,6 +155,24 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) self._assert_x_y(row_by_cartodb_id[2], -61.6961807, -29.5031057) + def test_template_with_two_columns_geocoding(self): + query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ + "cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + " 'select * from (' ||" \ + " ' select 1 as cartodb_id, ''Valladolid'' as city, ''Mexico'' as country ' ||" \ + " ' union all ' ||" \ + " ' select 2, ''Valladolid'', ''Spain''' ||" \ + " ') _x'," \ + "'city || '', '' || country')" + response = self._run_authenticated(query) + # from nose.tools import set_trace; set_trace() + + assert_equal(response['total_rows'], 2) + + row_by_cartodb_id = self._row_by_cartodb_id(response) + self._assert_x_y(row_by_cartodb_id[1], -88.2022488, 20.68964) + self._assert_x_y(row_by_cartodb_id[2], -4.7245321, 41.652251) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From 9c90c539f83551de2645522c22ccbd0c75d34be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 11:09:32 +0200 Subject: [PATCH 074/265] Fix Mapbox routing fixture shape --- server/lib/python/cartodb_services/test/test_mapboxrouting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_mapboxrouting.py b/server/lib/python/cartodb_services/test/test_mapboxrouting.py index 2871ab8..160b2a5 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapboxrouting.py @@ -25,7 +25,8 @@ WELL_KNOWN_SHAPE = [(40.73312, -73.98891), (40.73353, -73.98987), (40.73186, -73.99664), (40.73147, -73.99693), (40.73141, -73.99698), (40.73147, -73.99707), (40.73219, -73.99856), (40.73222, -73.99861), - (40.73293, -74.00007), (40.733, -74.00001)] + (40.73225, -73.99868), (40.73293, -74.00007), + (40.733, -74.00001)] WELL_KNOWN_LENGTH = 1317.9 From a931086e296b0546a8223a18260bb1c17d832249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 13:04:01 +0200 Subject: [PATCH 075/265] Log now also logs the exception, fixing fixtures --- .../test/expected/50_namedplaces_test.out | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index e24c4a0..6f18d8f 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,21 +1,21 @@ -- Check that the public function is callable, even with no data -- It should return NULL SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ @@ -35,42 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 From d0b04a97b8ff6f30e9b5a173aafa3d65b53dd6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 13:47:57 +0200 Subject: [PATCH 076/265] Better Google geocoding through concatenation --- .../python/cartodb_services/cartodb_services/google/geocoder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index ba0a663..a51c990 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -61,6 +61,8 @@ class GoogleMapsGeocoder: search_id, address, city, state, country = \ [search.get(k, None) for k in self.SEARCH_KEYS] opt_params = self._build_optional_parameters(city, state, country) + # Geocoding works better if components are also inside the address + address = ', '.join(filter(None, [address, city, state, country])) if address: self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) result = pool.apply_async(async_geocoder, From 8fe9903e7ae927468041f41005c67146df42d686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 15:14:11 +0200 Subject: [PATCH 077/265] searchtext -> searches for bulk geocoding --- client/cdb_dataservices_client--0.25.0.sql | 18 +++++++++--------- client/renderer/interface.yaml | 2 +- .../cdb_dataservices_server--0.32.0.sql | 6 +++--- .../extension/sql/21_bulk_geocode_street.sql | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 6900c64..be72fe5 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -420,7 +420,7 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -436,7 +436,7 @@ BEGIN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searchtext); + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- @@ -2490,7 +2490,7 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -2511,7 +2511,7 @@ BEGIN BEGIN - RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searchtext); + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, @@ -4451,12 +4451,12 @@ RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; -DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ CONNECT cdb_dataservices_client._server_conn_str(); - SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searchtext); + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); @@ -4966,8 +4966,8 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 22c3005..394a5fa 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -75,7 +75,7 @@ multi_row: true multi_field: true params: - - { name: searchtext, type: jsonb } + - { name: searches, type: jsonb } # Array of JSON objects with id, address, city, state and country fields - name: cdb_here_geocode_street_point return_type: Geometry diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 775c904..44f3d98 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2348,7 +2348,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -2363,12 +2363,12 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'searchtext': searchtext} + params = {'searches': searches} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) - result = plpy.execute(google_plan, [username, orgname, searchtext]) + result = plpy.execute(google_plan, [username, orgname, searches]) return result else: raise Exception('Requested geocoder is not available') diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index aebb6b8..8a9696e 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -5,7 +5,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -20,12 +20,12 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'searchtext': searchtext} + params = {'searches': searches} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) - result = plpy.execute(google_plan, [username, orgname, searchtext]) + result = plpy.execute(google_plan, [username, orgname, searches]) return result else: raise Exception('Requested geocoder is not available') From 379257b4b49f1e541d84bb1e0444c95dd504052b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 27 Jun 2018 19:07:19 +0200 Subject: [PATCH 078/265] Fix quota check --- client/sql/21_bulk_geocoding_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index b3ff689..4fbd5d1 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -24,7 +24,7 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF enough_quota IS NOT NULL AND enough_quota THEN + IF enough_quota IS NOT NULL AND NOT enough_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; From 45b8fc4ecf6903892b4ff6a43bdf636e65735898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 28 Jun 2018 13:06:52 +0200 Subject: [PATCH 079/265] Quota and batch size checks fixes and tests --- client/cdb_dataservices_client--0.25.0.sql | 9 +++++-- client/sql/21_bulk_geocoding_functions.sql | 7 ++++- .../21_bulk_geocoding_functions_test.out | 21 +++++++++++++++ .../sql/21_bulk_geocoding_functions_test.sql | 26 +++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 client/test/expected/21_bulk_geocoding_functions_test.out create mode 100644 client/test/sql/21_bulk_geocoding_functions_test.sql diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index be72fe5..2e906d7 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1992,6 +1992,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; + remaining_quota integer; cartodb_id_batch integer; batches_n integer; @@ -2002,7 +2003,7 @@ DECLARE temp_table_name text; BEGIN IF batch_size IS NULL THEN - batch_size := DEFAULT_BATCH_SIZE; + RAISE EXCEPTION 'batch_size can''t be null'; ELSIF batch_size > MAX_BATCH_SIZE THEN RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; END IF; @@ -2012,7 +2013,11 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF enough_quota IS NOT NULL AND enough_quota THEN + IF enough_quota IS NOT NULL AND NOT enough_quota THEN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota + INTO remaining_quota + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 4fbd5d1..4314e7c 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -4,6 +4,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; + remaining_quota integer; cartodb_id_batch integer; batches_n integer; @@ -14,7 +15,7 @@ DECLARE temp_table_name text; BEGIN IF batch_size IS NULL THEN - batch_size := DEFAULT_BATCH_SIZE; + RAISE EXCEPTION 'batch_size can''t be null'; ELSIF batch_size > MAX_BATCH_SIZE THEN RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; END IF; @@ -25,6 +26,10 @@ BEGIN query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; IF enough_quota IS NOT NULL AND NOT enough_quota THEN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota + INTO remaining_quota + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out new file mode 100644 index 0000000..7ca70db --- /dev/null +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -0,0 +1,21 @@ +\set VERBOSITY terse +-- Test bulk size mandatory +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ERROR: batch_size can't be null +-- Test quota check by mocking quota 0 +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT FALSE; +$$ LANGUAGE SQL; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; +$$ LANGUAGE SQL; +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +ERROR: Remaining quota: 0. Estimated cost: 1 +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql new file mode 100644 index 0000000..5433bd7 --- /dev/null +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -0,0 +1,26 @@ +\set VERBOSITY terse + +-- Test bulk size mandatory +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); + +-- Test quota check by mocking quota 0 +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT FALSE; +$$ LANGUAGE SQL; + +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; +$$ LANGUAGE SQL; + +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); + +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; + +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; + From e884b1d1f446e19eb65f0d3445452cb2d0bb0aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 28 Jun 2018 13:11:28 +0200 Subject: [PATCH 080/265] Fixture fix --- test/integration/test_street_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 5f8ce60..6634627 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -57,7 +57,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(response['total_rows'], 2) row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -3.7074009, 40.415511) + self._assert_x_y(row_by_cartodb_id[1], -4.728252, 41.6517025); self._assert_x_y(row_by_cartodb_id[2], -4.7404453, 41.6314339) def test_empty_columns(self): From 18e2349713bf8e71260a44d2fa9d282484974411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 29 Jun 2018 14:59:07 +0200 Subject: [PATCH 081/265] Bulk geocoding refactor --- server/__init__.py | 0 .../cdb_dataservices_server--0.32.0.sql | 56 +++++--------- .../extension/sql/21_bulk_geocode_street.sql | 58 +++++--------- server/lib/__init__.py | 0 server/lib/python/__init__.py | 0 .../lib/python/cartodb_services/__init__.py | 0 .../cartodb_services/__init__.py | 2 + .../cartodb_services/geocoder.py | 76 +++++++++++++++++++ .../cartodb_services/google/geocoder.py | 21 ++--- .../cartodb_services/here/geocoder.py | 10 ++- 10 files changed, 134 insertions(+), 89 deletions(-) create mode 100644 server/__init__.py create mode 100644 server/lib/__init__.py create mode 100644 server/lib/python/__init__.py create mode 100644 server/lib/python/cartodb_services/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/geocoder.py diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 44f3d98..1f77d9a 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2367,53 +2367,37 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) - result = plpy.execute(google_plan, [username, orgname, searches]) - return result + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + elif user_geocoder_config.heremaps_geocoder: + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) else: raise Exception('Requested geocoder is not available') + result = plpy.execute(plan, [username, orgname, searches]) + return result + $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ - from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager from cartodb_services.google import GoogleMapsGeocoder - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - - logger = Logger(logger_config) service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - try: - service_manager.assert_within_limits(quota=False) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) - geocode_results = geocoder.bulk_geocode(searches=searches) - if geocode_results: - results = [] - for result in geocode_results: - if result[1]: - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) - point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], None]) - else: - results.append([result[0], None, None]) - service_manager.quota_service.increment_success_service_use(len(results)) - return results - else: - service_manager.quota_service.increment_empty_service_use(len(searches)) - return [] - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use(len(searches)) - return [] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to bulk geocode street point using google maps') - finally: - service_manager.quota_service.increment_total_service_use() +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 8a9696e..0913117 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -24,52 +24,36 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) - result = plpy.execute(google_plan, [username, orgname, searches]) - return result + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + elif user_geocoder_config.heremaps_geocoder: + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) else: raise Exception('Requested geocoder is not available') + result = plpy.execute(plan, [username, orgname, searches]) + return result + $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ - from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager from cartodb_services.google import GoogleMapsGeocoder - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - - logger = Logger(logger_config) service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - - try: - service_manager.assert_within_limits(quota=False) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) - geocode_results = geocoder.bulk_geocode(searches=searches) - if geocode_results: - results = [] - for result in geocode_results: - if result[1]: - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) - point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], None]) - else: - results.append([result[0], None, None]) - service_manager.quota_service.increment_success_service_use(len(results)) - return results - else: - service_manager.quota_service.increment_empty_service_use(len(searches)) - return [] - except QuotaExceededException as qe: - service_manager.quota_service.increment_failed_service_use(len(searches)) - return [] - except BaseException as e: - import sys - service_manager.quota_service.increment_failed_service_use() - service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to bulk geocode street point using google maps') - finally: - service_manager.quota_service.increment_total_service_use() + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/__init__.py b/server/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/__init__.py b/server/lib/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/__init__.py b/server/lib/python/cartodb_services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index 61670c2..ef5caa4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -33,3 +33,5 @@ def _reset(): plpy = None GD = None + +from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py new file mode 100644 index 0000000..96679fd --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -0,0 +1,76 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +from tools import QuotaExceededException, Logger +from collections import namedtuple +import json + + +def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches): + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + + logger = Logger(logger_config) + + try: + service_manager.assert_within_limits(quota=False) + geocode_results = geocoder.bulk_geocode(searches=searches) + if geocode_results: + results = [] + for result in geocode_results: + if result[1]: + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) + point = plpy.execute(plan, result[1], 1)[0] + results.append([result[0], point['the_geom'], None]) + else: + results.append([result[0], None, None]) + service_manager.quota_service.increment_success_service_use(len(results)) + return results + else: + service_manager.quota_service.increment_empty_service_use(len(searches)) + return [] + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use(len(searches)) + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to bulk geocode street point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to bulk geocode street') + finally: + service_manager.quota_service.increment_total_service_use() + + +StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state country') + + +class StreetPointBulkGeocoder: + """ + Classes extending StreetPointBulkGeocoder should implement: + * _bulk_geocode(decoded_searches) + """ + + SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] + + def bulk_geocode(self, searches): + """ + :param searches: array of StreetGeocoderSearch + :return: array of tuples with three elements: + * id + * latitude and longitude (array of two elements) + * empty array (future use: metadata) + """ + try: + decoded_searches = json.loads(searches) + except Exception as e: + self._logger.error('General error', exception=e) + raise e + + street_geocoder_searches = [] + for search in decoded_searches: + search_id, address, city, state, country = \ + [search.get(k, None) for k in self.SEARCH_KEYS] + street_geocoder_searches.append( + (search_id, address, city, state, country)) + + return self._bulk_geocode(street_geocoder_searches) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index a51c990..1cc61c9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -5,13 +5,12 @@ import googlemaps from urlparse import parse_qs from exceptions import MalformedResult +from cartodb_services import StreetPointBulkGeocoder from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory from multiprocessing import Pool, TimeoutError -import json - import time, random def async_geocoder(geocoder, address, components): @@ -22,10 +21,9 @@ def async_geocoder(geocoder, address, components): results = geocoder.geocode(address=address, components=components) return results if results else [] -class GoogleMapsGeocoder: +class GoogleMapsGeocoder(StreetPointBulkGeocoder): """A Google Maps Geocoder wrapper for python""" PARALLEL_PROCESSES = 13 - SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] def __init__(self, client_id, client_secret, logger): if client_id is None: @@ -48,18 +46,11 @@ class GoogleMapsGeocoder: except KeyError: raise MalformedResult() - def bulk_geocode(self, searches): - try: - decoded_searches = json.loads(searches) - except Exception as e: - self._logger.error('General error', exception=e) - raise e - + def _bulk_geocode(self, searches): bulk_results = {} pool = Pool(processes=self.PARALLEL_PROCESSES) - for search in decoded_searches: - search_id, address, city, state, country = \ - [search.get(k, None) for k in self.SEARCH_KEYS] + for search in searches: + (search_id, address, city, state, country) = search opt_params = self._build_optional_parameters(city, state, country) # Geocoding works better if components are also inside the address address = ', '.join(filter(None, [address, city, state, country])) @@ -83,7 +74,7 @@ class GoogleMapsGeocoder: result = [] lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] - results.append([search_id, lng_lat, []]) + results.append((search_id, lng_lat, [])) return results except KeyError as e: self._logger.error('KeyError error', exception=e) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 6c3b81c..b72f77c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,10 +6,11 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * +from cartodb_services import StreetPointBulkGeocoder from cartodb_services.metrics import Traceable -class HereMapsGeocoder(Traceable): +class HereMapsGeocoder(Traceable, StreetPointBulkGeocoder): 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -64,6 +65,13 @@ class HereMapsGeocoder(Traceable): self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) + def _bulk_geocode(self, searches): + results = [] + for search in searches: + result = () + return results + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + def geocode(self, **kwargs): params = {} for key, value in kwargs.iteritems(): From fc610313bf807911cdf741b8b0aeafce4c8f122c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 29 Jun 2018 19:18:53 +0200 Subject: [PATCH 082/265] Test refactor and Here serial batch --- .../cartodb_services/here/geocoder.py | 5 +- test/integration/test_street_functions.py | 126 ++++++++++++------ 2 files changed, 85 insertions(+), 46 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index b72f77c..efef8b1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -68,9 +68,10 @@ class HereMapsGeocoder(Traceable, StreetPointBulkGeocoder): def _bulk_geocode(self, searches): results = [] for search in searches: - result = () + (search_id, address, city, state, country) = search + coordinates = self.geocode(searchtext=address, city=city, state=state, country=country) + results.append((search_id, coordinates, [])) return results - coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) def geocode(self, **kwargs): params = {} diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 6634627..db80f75 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -40,6 +40,52 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): class TestBulkStreetFunctions(TestStreetFunctionsSetUp): + provider = None + fixture_points = None + + GOOGLE_POINTS = { + 'Plaza Mayor, Valladolid': [-4.728252, 41.6517025], + 'Paseo Zorrilla, Valladolid': [-4.7404453, 41.6314339], + '1900 amphitheatre parkway': [-122.0875324, 37.4227968], + '1901 amphitheatre parkway': [-122.0885504, 37.4238657], + '1902 amphitheatre parkway': [-122.0876674, 37.4235729], + 'Valladolid': [-4.7245321, 41.652251], + 'Valladolid, Spain': [-4.7245321, 41.652251], + 'Valladolid, Mexico': [-88.2022488, 20.68964], + 'Madrid': [-3.7037902, 40.4167754], + 'Logroño, Spain': [-2.4449852, 42.4627195], + 'Logroño, Argentina': [-61.6961807, -29.5031057] + } + + HERE_POINTS = { + 'Plaza Mayor, Valladolid': [-4.72979, 41.65258], + 'Paseo Zorrilla, Valladolid': [-4.73869, 41.63817], + '1900 amphitheatre parkway': [-122.0879468, 37.4234763], + '1901 amphitheatre parkway': [-122.0879253, 37.4238725], + '1902 amphitheatre parkway': [-122.0879531, 37.4234775], + 'Valladolid': [-4.73214, 41.6542], + 'Valladolid, Spain': [-4.73214, 41.6542], + 'Valladolid, Mexico': [-88.20117, 20.69021], + 'Madrid': [-3.70578, 40.42028], + 'Logroño, Spain': [-2.45194, 42.46592], + 'Logroño, Argentina': [-61.69604, -29.50425] + } + + FIXTURE_POINTS = { + 'google': GOOGLE_POINTS, + 'heremaps': HERE_POINTS + } + + def setUp(self): + TestStreetFunctionsSetUp.setUp(self) + + if not self.fixture_points: + query = "select provider from " \ + "cdb_dataservices_client.cdb_service_quota_info() " \ + "where service = 'hires_geocoder'" + response = self._run_authenticated(query) + provider = response['rows'][0]['provider'] + self.fixture_points = self.FIXTURE_POINTS[provider] def test_full_spec(self): query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ @@ -54,11 +100,11 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): ", 'street', 'city', 'state', 'country')" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 2) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -4.728252, 41.6517025); - self._assert_x_y(row_by_cartodb_id[2], -4.7404453, 41.6314339) + points_by_cartodb_id = { + 1: self.fixture_points['Plaza Mayor, Valladolid'], + 2: self.fixture_points['Paseo Zorrilla, Valladolid'] + } + assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_empty_columns(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -69,10 +115,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'address', '''''', '''''', '''''')" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 1) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -122.0885504, 37.4238657) + assert_equal(self._x_y_by_cartodb_id(response)[1], + self.fixture_points['1901 amphitheatre parkway']) def test_null_columns(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -83,10 +127,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'address')" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 1) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -122.0885504, 37.4238657) + assert_equal(self._x_y_by_cartodb_id(response)[1], + self.fixture_points['1901 amphitheatre parkway']) def test_batching(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -99,12 +141,12 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'address', null, null, null, 2)" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 3) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -122.0875324, 37.4227968) - self._assert_x_y(row_by_cartodb_id[2], -122.0885504, 37.4238657) - self._assert_x_y(row_by_cartodb_id[3], -122.0876674, 37.4235729) + points_by_cartodb_id = { + 1: self.fixture_points['1900 amphitheatre parkway'], + 2: self.fixture_points['1901 amphitheatre parkway'], + 3: self.fixture_points['1902 amphitheatre parkway'], + } + assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_city_column_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -118,9 +160,11 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(response['total_rows'], 2) - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -4.7245321, 41.652251) - self._assert_x_y(row_by_cartodb_id[2], -3.7037902, 40.4167754) + points_by_cartodb_id = { + 1: self.fixture_points['Valladolid'], + 2: self.fixture_points['Madrid'] + } + assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_free_text_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -132,10 +176,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'''Logroño, La Rioja, Spain''')" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 1) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) + assert_equal(self._x_y_by_cartodb_id(response)[1], + self.fixture_points['Logroño, Spain']) def test_templating_geocoding(self): query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ @@ -149,11 +191,11 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'city || '', '' || ''Argentina''')" response = self._run_authenticated(query) - assert_equal(response['total_rows'], 2) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -2.4449852, 42.4627195) - self._assert_x_y(row_by_cartodb_id[2], -61.6961807, -29.5031057) + points_by_cartodb_id = { + 1: self.fixture_points['Logroño, Spain'], + 2: self.fixture_points['Logroño, Argentina'] + } + assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_template_with_two_columns_geocoding(self): query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ @@ -165,14 +207,12 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): " ') _x'," \ "'city || '', '' || country')" response = self._run_authenticated(query) - # from nose.tools import set_trace; set_trace() - - assert_equal(response['total_rows'], 2) - - row_by_cartodb_id = self._row_by_cartodb_id(response) - self._assert_x_y(row_by_cartodb_id[1], -88.2022488, 20.68964) - self._assert_x_y(row_by_cartodb_id[2], -4.7245321, 41.652251) + points_by_cartodb_id = { + 1: self.fixture_points['Valladolid, Mexico'], + 2: self.fixture_points['Valladolid, Spain'] + } + assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, @@ -181,10 +221,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): return IntegrationTestHelper.execute_query_raw(self.sql_api_url, authenticated_query) - def _row_by_cartodb_id(self, response): - return {r['cartodb_id']: r for r in response['rows']} - - def _assert_x_y(self, row, expected_x, expected_y): - assert_equal(row['st_x'], expected_x) - assert_equal(row['st_y'], expected_y) + @staticmethod + def _x_y_by_cartodb_id(response): + return {r['cartodb_id']: [r['st_x'], r['st_y']] + for r in response['rows']} From e416a8a641071183a1313d90dca2dff65f7d2ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 2 Jul 2018 18:35:36 +0200 Subject: [PATCH 083/265] HERE batch support --- client/cdb_dataservices_client--0.25.0.sql | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- .../cdb_dataservices_server--0.32.0.sql | 4 +- .../extension/sql/21_bulk_geocode_street.sql | 4 +- .../cartodb_services/geocoder.py | 2 +- .../cartodb_services/here/__init__.py | 1 + .../cartodb_services/here/bulk_geocoder.py | 153 ++++++++++++++++++ .../cartodb_services/here/geocoder.py | 11 +- test/integration/test_street_functions.py | 19 +++ 9 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 2e906d7..0742f43 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1997,7 +1997,7 @@ DECLARE cartodb_id_batch integer; batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; - MAX_BATCH_SIZE CONSTANT numeric := 1000; + MAX_BATCH_SIZE CONSTANT numeric := 10000; current_row_count integer ; temp_table_name text; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 4314e7c..d0bcf5e 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -9,7 +9,7 @@ DECLARE cartodb_id_batch integer; batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; - MAX_BATCH_SIZE CONSTANT numeric := 1000; + MAX_BATCH_SIZE CONSTANT numeric := 10000; current_row_count integer ; temp_table_name text; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 1f77d9a..849c688 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2393,10 +2393,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_st RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services import run_street_point_geocoder from cartodb_services.tools import LegacyServiceManager - from cartodb_services.here import HereMapsGeocoder + from cartodb_services.here import HereMapsBulkGeocoder service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 0913117..6732340 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -50,10 +50,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_st RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services import run_street_point_geocoder from cartodb_services.tools import LegacyServiceManager - from cartodb_services.here import HereMapsGeocoder + from cartodb_services.here import HereMapsBulkGeocoder service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 96679fd..d82768a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -71,6 +71,6 @@ class StreetPointBulkGeocoder: search_id, address, city, state, country = \ [search.get(k, None) for k in self.SEARCH_KEYS] street_geocoder_searches.append( - (search_id, address, city, state, country)) + StreetGeocoderSearch(search_id, address, city, state, country)) return self._bulk_geocode(street_geocoder_searches) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py index 7c20cc5..85cc596 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py @@ -1,2 +1,3 @@ from geocoder import HereMapsGeocoder +from bulk_geocoder import HereMapsBulkGeocoder from routing import HereMapsRoutingIsoline diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py new file mode 100644 index 0000000..3a28968 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -0,0 +1,153 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + + +import requests, time, zipfile, io, csv, cStringIO +import xml.etree.ElementTree as ET +from collections import namedtuple +from requests.adapters import HTTPAdapter +from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.here import HereMapsGeocoder +from cartodb_services.metrics import Traceable +from cartodb_services.tools.exceptions import ServiceException + + +HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status') + +class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): + BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' + MAX_BATCH_SIZE = 1000000 # From the docs + # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html + META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] + MAX_STALLED_RETRIES = 100 + BATCH_RETRY_SLEEP_S = 5 + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed'] + + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS): + HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults) + self.session = requests.Session() + self.session.mount(self.BATCH_URL, + HTTPAdapter(max_retries=self.max_retries)) + self.credentials_params = { + 'app_id': self.app_id, + 'app_code': self.app_code, + } + + def _bulk_geocode(self, searches): + if len(searches) > self.MAX_STALLED_RETRIES: + raise "Batch size can't be larger than {}".format(self.MAX_STALLED_RETRIES) + if self._should_use_batch(searches): + self._logger.debug('--> Batch geocode') + return self._batch_geocode(searches) + else: + self._logger.debug('--> Serial geocode') + return self._serial_geocode(searches) + + def _should_use_batch(self, searches): + return len(searches) >= self.MIN_BATCHED_SEARCH + + def _serial_geocode(self, searches): + results = [] + for search in searches: + (search_id, address, city, state, country) = search + coordinates = self.geocode(searchtext=address, city=city, state=state, country=country) + results.append((search_id, coordinates, [])) + return results + + def _batch_geocode(self, searches): + request_id = self._send_batch(self._searches_to_csv(searches)) + self._logger.debug('--> Sent batch {}'.format(request_id)) + + last_processed = 0 + stalled_retries = 0 + # https://developer.here.com/documentation/batch-geocoder/topics/job-status.html + while True: + job_info = self._job_status(request_id) + if job_info.processed_count == last_processed: + stalled_retries += 1 + if stalled_retries > self.MAX_STALLED_RETRIES: + raise Exception('Too many retries for job {}'.format(request_id)) + else: + stalled_retries = 0 + last_processed = job_info.processed_count + + self._logger.debug('--> Job poll check: {}'.format(job_info)) + if job_info.status in self.JOB_FINAL_STATES: + break + else: + time.sleep(self.BATCH_RETRY_SLEEP_S) + + self._logger.debug('--> Job complete: {}'.format(job_info)) + + results = self._download_results(request_id) + self._logger.debug('--> Results: {} rows; {}'.format(len(results), results)) + + return results + + def _searches_to_csv(self, searches): + queue = cStringIO.StringIO() + writer = csv.writer(queue, delimiter='|') + writer.writerow(['recId', 'searchText', 'country']) + + for search in searches: + fields = [search.address, search.city, search.state] + search_text = ', '.join(filter(None, fields)) + row = [s.encode("utf-8") + for s in [str(search.id), search_text, search.country]] + writer.writerow(row) + + return queue.getvalue() + + def _send_batch(self, data): + cols = 'displayLatitude,displayLongitude,' + ','.join(self.META_COLS) + request_params = self.credentials_params.copy() + request_params.update({ + 'gen': 8, + 'action': 'run', + #'mailto': 'juanignaciosl@carto.com', + 'header': 'true', + 'inDelim': '|', + 'outDelim': '|', + 'outCols': cols, + 'outputcombined': 'true' + }) + + response = self.session.post(self.BATCH_URL, data=data, + params=request_params, + timeout=(self.connect_timeout, self.read_timeout)) + + if response.status_code == 200: + root = ET.fromstring(response.text) + return root.find('./Response/MetaInfo/RequestId').text + else: + raise ServiceException("Error sending HERE batch", response) + + def _job_status(self, request_id): + polling_params = self.credentials_params.copy() + polling_params.update({'action': 'status'}) + polling_r = self.session.get("{}/{}".format(self.BATCH_URL, request_id), + params=polling_params, + timeout=(self.connect_timeout, self.read_timeout)) + polling_root = ET.fromstring(polling_r.text) + return HereJobStatus( + total_count=polling_root.find('./Response/TotalCount').text, + processed_count=polling_root.find('./Response/ProcessedCount').text, + status=polling_root.find('./Response/Status').text) + + def _download_results(self, job_id): + result_r = self.session.get("{}/{}/result".format(self.BATCH_URL, job_id), + params=self.credentials_params, + timeout=(self.connect_timeout, self.read_timeout)) + root_zip = zipfile.ZipFile(io.BytesIO(result_r.content)) + + results = [] + for name in root_zip.namelist(): + if name.endswith('_out.txt'): + reader = csv.DictReader(root_zip.open(name), delimiter='|') + for row in reader: + if row['SeqNumber'] == '1': # First per requested data + results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']])) + + return results + diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index efef8b1..6c3b81c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,11 +6,10 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * -from cartodb_services import StreetPointBulkGeocoder from cartodb_services.metrics import Traceable -class HereMapsGeocoder(Traceable, StreetPointBulkGeocoder): +class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -65,14 +64,6 @@ class HereMapsGeocoder(Traceable, StreetPointBulkGeocoder): self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) - def _bulk_geocode(self, searches): - results = [] - for search in searches: - (search_id, address, city, state, country) = search - coordinates = self.geocode(searchtext=address, city=city, state=state, country=country) - results.append((search_id, coordinates, [])) - return results - def geocode(self, **kwargs): params = {} for key, value in kwargs.iteritems(): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index db80f75..45bda56 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -214,6 +214,25 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): } assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + def test_large_batches(self): + """ + Useful just to test a good batch size + """ + n = 10 + streets = [] + for i in range(0, n): + streets.append('{{"cartodb_id": {}, "address": "{} Yonge Street, ' \ + 'Toronto, Canada"}}'.format(i, i)) + + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "'address', null, null, null, {})".format(','.join(streets), n) + response = self._run_authenticated(query) + assert_equal(n - 1, len(response['rows'])) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From 9856adb7ce6ed9d254473b629ec8603f276e74e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 4 Jul 2018 12:32:52 +0200 Subject: [PATCH 084/265] Explicit NotImplementedError --- .../python/cartodb_services/cartodb_services/geocoder.py | 8 +++++++- .../cartodb_services/here/bulk_geocoder.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index d82768a..d3c1c60 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -47,7 +47,7 @@ StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state class StreetPointBulkGeocoder: """ Classes extending StreetPointBulkGeocoder should implement: - * _bulk_geocode(decoded_searches) + * _bulk_geocode(street_geocoder_searches) """ SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] @@ -74,3 +74,9 @@ class StreetPointBulkGeocoder: StreetGeocoderSearch(search_id, address, city, state, country)) return self._bulk_geocode(street_geocoder_searches) + + def _bulk_geocode(self, street_geocoder_searches): + """ + Subclasses must implement _bulk_geocode + """ + raise NotImplementedError('Subclasses must implement _bulk_geocode') diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 3a28968..dfc017b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -16,12 +16,12 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status' class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' - MAX_BATCH_SIZE = 1000000 # From the docs + MAX_BATCH_SIZE = 1000000 # From the docs # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] MAX_STALLED_RETRIES = 100 BATCH_RETRY_SLEEP_S = 5 - MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed'] def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS): From 754c364d22e773b00a763125609e637c828850d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 4 Jul 2018 13:24:11 +0200 Subject: [PATCH 085/265] Reduce precision on fixture points --- test/helpers/integration_test_helper.py | 25 +++++++++++++++++++++++ test/integration/test_street_functions.py | 22 ++++++++++++-------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/test/helpers/integration_test_helper.py b/test/helpers/integration_test_helper.py index 7fdcd38..df050de 100644 --- a/test/helpers/integration_test_helper.py +++ b/test/helpers/integration_test_helper.py @@ -2,6 +2,29 @@ import os import requests import json +from nose.tools import assert_true + + +# From https://www.python.org/dev/peps/pep-0485/#proposed-implementation +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + +def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005): + """ + Asserts that the given points are "close enough", in a square. + :param xy_a: Array of 2 elements, X and Y. + :param xy_b: Array of 2 elements, X and Y. + :param rel_tol: Relative tolerance. Default: 0.001 (0.1%). + :param abs_tol: Absolute tolerance. Default: 0.0005. + """ + + for i in [0, 1]: + assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol), + "Coord {} error: {} and {} are not closer than {}, {}".format( + i, xy_a[i], xy_b[i], rel_tol, abs_tol + )) + class IntegrationTestHelper: @@ -34,3 +57,5 @@ class IntegrationTestHelper: @classmethod def execute_query(cls, sql_api_url, query): return cls.execute_query_raw(sql_api_url, query)['rows'][0] + + diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 45bda56..420e6a2 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -5,7 +5,7 @@ from unittest import TestCase from nose.tools import assert_raises from nose.tools import assert_not_equal, assert_equal from ..helpers.integration_test_helper import IntegrationTestHelper - +from ..helpers.integration_test_helper import assert_close_enough class TestStreetFunctionsSetUp(TestCase): @@ -104,7 +104,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 1: self.fixture_points['Plaza Mayor, Valladolid'], 2: self.fixture_points['Paseo Zorrilla, Valladolid'] } - assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_empty_columns(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -115,7 +115,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'address', '''''', '''''', '''''')" response = self._run_authenticated(query) - assert_equal(self._x_y_by_cartodb_id(response)[1], + assert_close_enough(self._x_y_by_cartodb_id(response)[1], self.fixture_points['1901 amphitheatre parkway']) def test_null_columns(self): @@ -127,7 +127,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'address')" response = self._run_authenticated(query) - assert_equal(self._x_y_by_cartodb_id(response)[1], + assert_close_enough(self._x_y_by_cartodb_id(response)[1], self.fixture_points['1901 amphitheatre parkway']) def test_batching(self): @@ -146,7 +146,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 2: self.fixture_points['1901 amphitheatre parkway'], 3: self.fixture_points['1902 amphitheatre parkway'], } - assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_city_column_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -164,7 +164,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 1: self.fixture_points['Valladolid'], 2: self.fixture_points['Madrid'] } - assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_free_text_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ @@ -176,7 +176,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'''Logroño, La Rioja, Spain''')" response = self._run_authenticated(query) - assert_equal(self._x_y_by_cartodb_id(response)[1], + assert_close_enough(self._x_y_by_cartodb_id(response)[1], self.fixture_points['Logroño, Spain']) def test_templating_geocoding(self): @@ -195,7 +195,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 1: self.fixture_points['Logroño, Spain'], 2: self.fixture_points['Logroño, Argentina'] } - assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_template_with_two_columns_geocoding(self): query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \ @@ -212,7 +212,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 1: self.fixture_points['Valladolid, Mexico'], 2: self.fixture_points['Valladolid, Spain'] } - assert_equal(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) def test_large_batches(self): """ @@ -245,3 +245,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): return {r['cartodb_id']: [r['st_x'], r['st_y']] for r in response['rows']} + @staticmethod + def assert_close_points(points_a_by_cartodb_id, points_b_by_cartodb_id): + for cartodb_id, point in points_a_by_cartodb_id.iteritems(): + assert_close_enough(point, points_b_by_cartodb_id[cartodb_id]) From 44744de73d70c25c9d8c10802ac3e7c6152d6706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 4 Jul 2018 13:29:37 +0200 Subject: [PATCH 086/265] Explicit check for result length --- server/lib/python/cartodb_services/cartodb_services/geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index d3c1c60..6a14594 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -18,7 +18,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org if geocode_results: results = [] for result in geocode_results: - if result[1]: + if result[1] and len(result[1]) == 2: plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] results.append([result[0], point['the_geom'], None]) From 8968f0e6ec5bf5b03bef393c5f09e699dfd7c363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 4 Jul 2018 13:36:18 +0200 Subject: [PATCH 087/265] Fix message --- .../cartodb_services/cartodb_services/here/bulk_geocoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index dfc017b..0942394 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -35,8 +35,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): } def _bulk_geocode(self, searches): - if len(searches) > self.MAX_STALLED_RETRIES: - raise "Batch size can't be larger than {}".format(self.MAX_STALLED_RETRIES) + if len(searches) > self.MAX_BATCH_SIZE: + raise "Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE) if self._should_use_batch(searches): self._logger.debug('--> Batch geocode') return self._batch_geocode(searches) From 89e9bf1ed6fce9d5b25b32ec4af0ff443bfe844a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 4 Jul 2018 15:07:54 +0200 Subject: [PATCH 088/265] Revert expected warnings --- .../test/expected/50_namedplaces_test.out | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index 6f18d8f..e24c4a0 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,21 +1,21 @@ -- Check that the public function is callable, even with no data -- It should return NULL SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ------------------------------ @@ -35,42 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 From 5b46c1527e9df5607d1edc2b39c0747f8b96780d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 08:23:33 +0200 Subject: [PATCH 089/265] Revert "Revert expected warnings" This reverts commit 89e9bf1ed6fce9d5b25b32ec4af0ff443bfe844a. --- .../test/expected/50_namedplaces_test.out | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index e24c4a0..6f18d8f 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,21 +1,21 @@ -- Check that the public function is callable, even with no data -- It should return NULL SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ------------------------------ @@ -35,42 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 From 3c071339129f319d7dcb9c5cf8b9cbba92481b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 08:48:20 +0200 Subject: [PATCH 090/265] Improve logging of exceptions from sys.exc_info --- .../test/expected/50_namedplaces_test.out | 18 +++++++++--------- .../cartodb_services/tools/log.py | 9 ++++++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index 6f18d8f..2e41285 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,21 +1,21 @@ -- Check that the public function is callable, even with no data -- It should return NULL SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ @@ -35,42 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: (, ExternalRoutineException("spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'",), ) +WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index 817a423..500c8ab 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -86,7 +86,14 @@ class Logger: self._file_logger.error(text, extra=extra_data) def _send_to_plpy(self, level, text, exception=None): - exception_message = '. Exception: {}'.format(str(exception)) if exception else '' + # exception might also be a tuple generated by sys.exc_info + if exception: + if isinstance(exception, tuple) and len(exception) > 1: + exception = exception[1] + exception_message = '. Exception: {}'.format(exception) + else: + exception_message = '' + message = '{}{}'.format(text, exception_message) if self._check_plpy(): if level == 'debug': From 6c89ca8d708055d925200216ad4fbb01cbb82e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 12:42:18 +0200 Subject: [PATCH 091/265] Fix exception raising --- .../cartodb_services/cartodb_services/here/bulk_geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 0942394..260441b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -36,7 +36,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): def _bulk_geocode(self, searches): if len(searches) > self.MAX_BATCH_SIZE: - raise "Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE) + raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) if self._should_use_batch(searches): self._logger.debug('--> Batch geocode') return self._batch_geocode(searches) From 23e3de9da5d244d92882da8c085605ec1bbe70ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 15:10:05 +0200 Subject: [PATCH 092/265] Add missing permission for cdb_bulk_geocode_street_point --- client/sql/95_grant_execute_manual.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/client/sql/95_grant_execute_manual.sql b/client/sql/95_grant_execute_manual.sql index 2ffd68f..8d6a013 100644 --- a/client/sql/95_grant_execute_manual.sql +++ b/client/sql/95_grant_execute_manual.sql @@ -1,3 +1,4 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; \ No newline at end of file From 91012ea62d6fc834bacc16acd1595548837d7191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 16:26:28 +0200 Subject: [PATCH 093/265] Updated version script --- client/cdb_dataservices_client--0.25.0.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 0742f43..e810dbc 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -5123,3 +5123,4 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_ex GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; \ No newline at end of file From d00a48f16e291eb95690436f06596ff19d22b28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 17:57:17 +0200 Subject: [PATCH 094/265] Check array length --- test/integration/test_street_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 420e6a2..1bf10cc 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -247,5 +247,6 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): @staticmethod def assert_close_points(points_a_by_cartodb_id, points_b_by_cartodb_id): + assert_equal(len(points_a_by_cartodb_id), len(points_b_by_cartodb_id)) for cartodb_id, point in points_a_by_cartodb_id.iteritems(): assert_close_enough(point, points_b_by_cartodb_id[cartodb_id]) From 6da70fd8ea40da191d14d95f94659f05518a28da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 6 Jul 2018 15:52:10 +0200 Subject: [PATCH 095/265] Fix encoding of missing fields --- .../cartodb_services/here/bulk_geocoder.py | 2 +- test/integration/test_street_functions.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 260441b..9a688f4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -93,7 +93,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): for search in searches: fields = [search.address, search.city, search.state] search_text = ', '.join(filter(None, fields)) - row = [s.encode("utf-8") + row = [s.encode("utf-8") if s else '' for s in [str(search.id), search_text, search.country]] writer.writerow(row) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 1bf10cc..f4a83b2 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -233,6 +233,13 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): response = self._run_authenticated(query) assert_equal(n - 1, len(response['rows'])) + def test_missing_components_on_private_function(self): + query = "SELECT _cdb_bulk_geocode_street_point(" \ + " '[{\"id\": \"1\", \"address\": \"Amphitheatre Parkway 22\"}]' " \ + ")" + response = self._run_authenticated(query) + assert_equal(1, len(response['rows'])) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From 5be43e15c0ad2d4967b9c43f1d8565681c6901fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 16:13:44 +0200 Subject: [PATCH 096/265] Fix error message on TomTom error --- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 432dd31..225f60d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -97,7 +97,8 @@ class TomTomGeocoder(Traceable): elif response.status_code == requests.codes.unprocessable_entity: return [] else: - raise ServiceException(response.status_code, response) + msg = 'Unknown response: {}'.format(str(response.status_code)) + raise ServiceException(msg, response) except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down From 31afc82b566c2c25e5677428485425521a419c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 5 Jul 2018 17:12:22 +0200 Subject: [PATCH 097/265] TomTom bulk geocoding by bypassing to serial --- server/extension/Makefile | 3 ++ .../cdb_dataservices_server--0.32.0.sql | 24 +++++++++++-- .../extension/sql/21_bulk_geocode_street.sql | 24 +++++++++++-- .../cartodb_services/tomtom/__init__.py | 1 + .../cartodb_services/tomtom/bulk_geocoder.py | 36 +++++++++++++++++++ test/integration/test_street_functions.py | 15 ++++++-- 6 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py diff --git a/server/extension/Makefile b/server/extension/Makefile index 8f53576..968e51c 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -83,3 +83,6 @@ deploy: release_remove_parallel_deploy $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' install: deploy + +reinstall: install + psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;" diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 849c688..4f66a15 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2367,12 +2367,15 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: - plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + provider_function = "_cdb_bulk_google_geocode_street_point"; elif user_geocoder_config.heremaps_geocoder: - plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; else: raise Exception('Requested geocoder is not available') + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) result = plpy.execute(plan, [username, orgname, searches]) return result @@ -2400,6 +2403,23 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 6732340..50d5008 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -24,12 +24,15 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: - plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + provider_function = "_cdb_bulk_google_geocode_street_point"; elif user_geocoder_config.heremaps_geocoder: - plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; else: raise Exception('Requested geocoder is not available') + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) result = plpy.execute(plan, [username, orgname, searches]) return result @@ -57,3 +60,20 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py index 8c5df3c..21d5ced 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py @@ -1,3 +1,4 @@ from geocoder import TomTomGeocoder +from bulk_geocoder import TomTomBulkGeocoder from routing import TomTomRouting, TomTomRoutingResponse from isolines import TomTomIsolines, TomTomIsochronesResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py new file mode 100644 index 0000000..34d0974 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -0,0 +1,36 @@ +from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.tomtom import TomTomGeocoder + + +class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): + # TODO: ? + MAX_BATCH_SIZE = 1000000 # From the docs + # TODO: ? + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + + def _bulk_geocode(self, searches): + if len(searches) > self.MAX_BATCH_SIZE: + raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) + if self._should_use_batch(searches): + self._logger.debug('--> Batch geocode') + return self._batch_geocode(searches) + else: + self._logger.debug('--> Serial geocode') + return self._serial_geocode(searches) + + def _should_use_batch(self, searches): + return len(searches) >= self.MIN_BATCHED_SEARCH + + def _serial_geocode(self, searches): + results = [] + for search in searches: + (search_id, address, city, state, country) = search + self._logger.debug('--> Sending serial search: {}'.format(search)) + coordinates = self.geocode(searchtext=address.encode('utf-8'), + city=city.encode('utf-8'), + state_province=state.encode('utf-8'), + country=country.encode('utf-8')) + self._logger.debug('--> result sent') + results.append((search_id, coordinates, [])) + return results + diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index f4a83b2..2060385 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -71,9 +71,20 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 'Logroño, Argentina': [-61.69604, -29.50425] } + TOMTOM_POINTS = HERE_POINTS.copy() + TOMTOM_POINTS.update({ + 'Plaza Mayor, Valladolid': [-4.72183, 41.5826], + 'Paseo Zorrilla, Valladolid': [-4.74031, 41.63181], + 'Valladolid': [-4.72838, 41.6542], + 'Valladolid, Spain': [-4.72838, 41.6542], + 'Madrid': [-3.70035, 40.42028], + 'Logroño, Spain': [-2.44998, 42.46592], + }) + FIXTURE_POINTS = { - 'google': GOOGLE_POINTS, - 'heremaps': HERE_POINTS + 'google': GOOGLE_POINTS, + 'heremaps': HERE_POINTS, + 'tomtom': TOMTOM_POINTS } def setUp(self): From bf8b76b5fe4a079fd4405bfd615e493bca322e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 6 Jul 2018 20:04:28 +0200 Subject: [PATCH 098/265] maxresults depends on batch --- .../cartodb_services/cartodb_services/here/bulk_geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 9a688f4..d652891 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -24,7 +24,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): MIN_BATCHED_SEARCH = 100 # Under this, serial will be used JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed'] - def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS): + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=MAX_BATCH_SIZE): HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults) self.session = requests.Session() self.session.mount(self.BATCH_URL, From b8475bac304e7290cf6d939ca4f8ddfcc7fe659b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 6 Jul 2018 20:11:48 +0200 Subject: [PATCH 099/265] TomTom batch geocoding --- .../cartodb_services/tomtom/bulk_geocoder.py | 92 +++++++++++++++++-- .../cartodb_services/tomtom/geocoder.py | 65 +++++++------ 2 files changed, 122 insertions(+), 35 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index 34d0974..ec3949f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -1,12 +1,30 @@ +import json, requests, time +from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder from cartodb_services.tomtom import TomTomGeocoder +from cartodb_services.tools.exceptions import ServiceException class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): - # TODO: ? + BASE_URL = 'https://api.tomtom.com' + BATCH_URL = BASE_URL + '/search/2/batch.json' MAX_BATCH_SIZE = 1000000 # From the docs - # TODO: ? - MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + MAX_STALLED_RETRIES = 100 + BATCH_RETRY_SLEEP_S = 5 + MIN_BATCHED_SEARCH = 10 # Batch API is really fast + READ_TIMEOUT = 60 + CONNECT_TIMEOUT = 10 + MAX_RETRIES = 1 + + def __init__(self, apikey, logger, service_params=None): + TomTomGeocoder.__init__(self, apikey, logger, service_params) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) + self.session = requests.Session() + self.session.headers.update({'Content-Type': 'application/json'}) + self.session.mount(self.BATCH_URL, + HTTPAdapter(max_retries=self.max_retries)) def _bulk_geocode(self, searches): if len(searches) > self.MAX_BATCH_SIZE: @@ -25,12 +43,72 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (search_id, address, city, state, country) = search + address = address.encode('utf-8') if address else None + city = city.encode('utf-8') if city else None + state = state.encode('utf-8') if state else None + country = country.encode('utf-8') if country else None self._logger.debug('--> Sending serial search: {}'.format(search)) - coordinates = self.geocode(searchtext=address.encode('utf-8'), - city=city.encode('utf-8'), - state_province=state.encode('utf-8'), - country=country.encode('utf-8')) + coordinates = self.geocode(searchtext=address, city=city, + state_province=state, country=country) self._logger.debug('--> result sent') results.append((search_id, coordinates, [])) return results + def _batch_geocode(self, searches): + location = self._send_batch(searches) + xy_results = self._download_results(location) + results = [] + for s, r in zip(searches, xy_results): + results.append((s[0], r, [])) + self._logger.debug('--> results: {}'.format(results)) + return results + + def _send_batch(self, searches): + body = {'batchItems': [{'query': self._query(s)} for s in searches]} + self._logger.debug('--> {}; Body: {}'.format(self.BATCH_URL, body)) + request_params = { + 'key': self._apikey + } + response = self.session.post(self.BATCH_URL, data=json.dumps(body), + allow_redirects=False, + params=request_params, + timeout=(self.connect_timeout, self.read_timeout)) + self._logger.debug('--> response: {}'.format(response.status_code)) + if response.status_code == 303: + self._logger.debug(response.headers) + return response.headers['Location'] + else: + msg = "Error sending batch: {}; Headers: {}".format( + response.text.encode('utf-8'), response.headers) + self._logger.error(msg) + raise ServiceException(msg, response) + + def _download_results(self, location): + stalled_retries = 0 + while True: + response = self.session.get(self.BASE_URL + location) + if response.status_code == 200: + self._logger.debug('--> Results ready {}'.format(location)) + return self._parse_results(response.json()) + elif response.status_code == 202: + stalled_retries += 1 + if stalled_retries > self.MAX_STALLED_RETRIES: + raise Exception('Too many retries for job {}'.format(location)) + location = response.headers['Location'] + self._logger.debug('--> Waiting for {}'.format(location)) + time.sleep(self.BATCH_RETRY_SLEEP_S) + else: + msg = "Error downloading batch: {}; Headers: {}".format( + response.text.encode('utf-8'), response.headers) + self._logger.error(msg) + raise ServiceException(msg, response) + + def _query(self, search): + (search_id, address, city, state, country) = search + searchtext = ', '.join(filter(None, [address, city, state])) + return self._request_uri(searchtext=searchtext, countries=country) + + def _parse_results(self, json_body): + return [self._parse_response(item['statusCode'], item['response']) + for item in json_body['batchItems']] + diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 225f60d..1f63f58 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -9,10 +9,11 @@ from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.normalize import normalize -BASEURI = ('https://api.tomtom.com/search/2/geocode/' - '{searchtext}.JSON' - '?key={apiKey}' - '&limit=1') +HOST = 'https://api.tomtom.com' +API_BASEURI = '/search/2' +REQUEST_BASEURI = ('/geocode/' + '{searchtext}.json' + '?limit=1') ENTRY_RESULTS = 'results' ENTRY_POSITION = 'position' ENTRY_LON = 'lon' @@ -30,20 +31,15 @@ class TomTomGeocoder(Traceable): self._logger = logger def _uri(self, searchtext, countries=None): - baseuri = BASEURI + '&countrySet={}'.format(countries) \ - if countries else BASEURI - uri = URITemplate(baseuri).expand(apiKey=self._apikey, - searchtext=searchtext.encode('utf-8')) - return uri + return API_BASEURI + self._request_uri(searchtext, countries, self._apikey) - def _parse_geocoder_response(self, response): - json_response = json.loads(response) - - if json_response and json_response[ENTRY_RESULTS]: - result = json_response[ENTRY_RESULTS][0] - return self._extract_lng_lat_from_feature(result) - else: - return [] + def _request_uri(self, searchtext, countries=None, apiKey=None): + baseuri = REQUEST_BASEURI + if countries: + baseuri += '&countrySet={}'.format(countries) + baseuri = baseuri + '&key={apiKey}' if apiKey else baseuri + return URITemplate(baseuri).expand(apiKey=apiKey, + searchtext=searchtext.encode('utf-8')) def _extract_lng_lat_from_feature(self, result): position = result[ENTRY_POSITION] @@ -85,20 +81,11 @@ class TomTomGeocoder(Traceable): if state_province: address.append(normalize(state_province)) - uri = self._uri(searchtext=', '.join(address), countries=country) + uri = HOST + self._uri(searchtext=', '.join(address), countries=country) try: response = requests.get(uri) - - if response.status_code == requests.codes.ok: - return self._parse_geocoder_response(response.text) - elif response.status_code == requests.codes.bad_request: - return [] - elif response.status_code == requests.codes.unprocessable_entity: - return [] - else: - msg = 'Unknown response: {}'.format(str(response.status_code)) - raise ServiceException(msg, response) + return self._parse_response(response.status_code, response.text) except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down @@ -111,3 +98,25 @@ class TomTomGeocoder(Traceable): self._logger.error('Error connecting to TomTom geocoding server', exception=ce) return [] + + def _parse_response(self, status_code, text): + if status_code == requests.codes.ok: + return self._parse_geocoder_response(text) + elif status_code == requests.codes.bad_request: + return [] + elif status_code == requests.codes.unprocessable_entity: + return [] + else: + msg = 'Unknown response {}: {}'.format(str(status_code), text) + raise ServiceException(msg, None) + + def _parse_geocoder_response(self, response): + json_response = json.loads(response) \ + if type(response) != dict else response + + if json_response and json_response[ENTRY_RESULTS]: + result = json_response[ENTRY_RESULTS][0] + return self._extract_lng_lat_from_feature(result) + else: + return [] + From ae84122c3dda0fbd3471d858b404e77e45e33aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Jul 2018 15:35:37 +0200 Subject: [PATCH 100/265] countries --> country --- .../cartodb_services/tomtom/bulk_geocoder.py | 2 +- .../cartodb_services/tomtom/geocoder.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index ec3949f..8036667 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -106,7 +106,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): def _query(self, search): (search_id, address, city, state, country) = search searchtext = ', '.join(filter(None, [address, city, state])) - return self._request_uri(searchtext=searchtext, countries=country) + return self._request_uri(searchtext=searchtext, country=country) def _parse_results(self, json_body): return [self._parse_response(item['statusCode'], item['response']) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 1f63f58..b4e82cb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -30,13 +30,14 @@ class TomTomGeocoder(Traceable): self._apikey = apikey self._logger = logger - def _uri(self, searchtext, countries=None): - return API_BASEURI + self._request_uri(searchtext, countries, self._apikey) + def _uri(self, searchtext, country=None): + return HOST + API_BASEURI + \ + self._request_uri(searchtext, country, self._apikey) - def _request_uri(self, searchtext, countries=None, apiKey=None): + def _request_uri(self, searchtext, country=None, apiKey=None): baseuri = REQUEST_BASEURI - if countries: - baseuri += '&countrySet={}'.format(countries) + if country: + baseuri += '&countrySet={}'.format(country) baseuri = baseuri + '&key={apiKey}' if apiKey else baseuri return URITemplate(baseuri).expand(apiKey=apiKey, searchtext=searchtext.encode('utf-8')) @@ -81,7 +82,7 @@ class TomTomGeocoder(Traceable): if state_province: address.append(normalize(state_province)) - uri = HOST + self._uri(searchtext=', '.join(address), countries=country) + uri = self._uri(searchtext=', '.join(address), country=country) try: response = requests.get(uri) From f618e4aec3367588eb94da86f1f9bd534e57d89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Sat, 7 Jul 2018 12:14:33 +0200 Subject: [PATCH 101/265] Mapbox bulk geocoding --- .../cdb_dataservices_server--0.32.0.sql | 19 ++++ .../extension/sql/21_bulk_geocode_street.sql | 19 ++++ .../cartodb_services/mapbox/__init__.py | 1 + .../cartodb_services/mapbox/bulk_geocoder.py | 90 +++++++++++++++++++ .../cartodb_services/mapbox/geocoder.py | 43 ++++++--- test/integration/test_street_functions.py | 13 ++- 6 files changed, 174 insertions(+), 11 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 4f66a15..8543e7f 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2372,6 +2372,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ provider_function = "_cdb_bulk_heremaps_geocode_street_point"; elif user_geocoder_config.tomtom_geocoder: provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; else: raise Exception('Requested geocoder is not available') @@ -2420,6 +2422,23 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 50d5008..adee498 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -29,6 +29,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ provider_function = "_cdb_bulk_heremaps_geocode_street_point"; elif user_geocoder_config.tomtom_geocoder: provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; else: raise Exception('Requested geocoder is not available') @@ -77,3 +79,20 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py index c613cdc..10ca46e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py @@ -1,4 +1,5 @@ from routing import MapboxRouting, MapboxRoutingResponse from geocoder import MapboxGeocoder +from bulk_geocoder import MapboxBulkGeocoder from isolines import MapboxIsolines, MapboxIsochronesResponse from matrix_client import MapboxMatrixClient diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py new file mode 100644 index 0000000..cb27f22 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -0,0 +1,90 @@ +import json, requests, time +from requests.adapters import HTTPAdapter +from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.mapbox import MapboxGeocoder +from cartodb_services.tools.exceptions import ServiceException +from iso3166 import countries +from cartodb_services.tools.country import country_to_iso3 + + +class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): + MAX_BATCH_SIZE = 50 # From the docs + MIN_BATCHED_SEARCH = 0 + READ_TIMEOUT = 60 + CONNECT_TIMEOUT = 10 + MAX_RETRIES = 1 + + def __init__(self, token, logger, service_params=None): + MapboxGeocoder.__init__(self, token, logger, service_params) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) + self.session = requests.Session() + + def _bulk_geocode(self, searches): + if len(searches) > self.MAX_BATCH_SIZE: + raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) + if self._should_use_batch(searches): + self._logger.debug('--> Batch geocode') + return self._batch_geocode(searches) + else: + self._logger.debug('--> Serial geocode') + return self._serial_geocode(searches) + + def _should_use_batch(self, searches): + return len(searches) >= self.MIN_BATCHED_SEARCH + + def _serial_geocode(self, searches): + results = [] + for search in searches: + elements = self._encoded_elements(search) + self._logger.debug('--> Sending serial search: {}'.format(search)) + coordinates = self._geocode_search(*elements) + results.append((search[0], coordinates, [])) + return results + + def _geocode_search(self, address, city, state, country): + coordinates = self.geocode(searchtext=address, city=city, + state_province=state, country=country) + self._logger.debug('--> result sent') + return coordinates + + def _encoded_elements(self, search): + (search_id, address, city, state, country) = search + address = address.encode('utf-8') if address else None + city = city.encode('utf-8') if city else None + state = state.encode('utf-8') if state else None + country = self._country_code(country) if country else None + return address, city, state, country + + def _batch_geocode(self, searches): + if len(searches) == 1: + # A single search can take advantage of country fixing + search = searches[0] + elements = self._encoded_elements(search) + coordinates = self._geocode_search(*elements) + return [(search[0], coordinates, [])] + else: + frees = [] + for search in searches: + elements = self._encoded_elements(search) + free = ', '.join([elem for elem in elements if elem]) + frees.append(free) + + self._logger.debug('--> sending free search: {}'.format(frees)) + xy_results = self.geocode_free_text(frees) + results = [] + self._logger.debug('--> searches: {}; xy: {}'.format(searches, xy_results)) + for s, r in zip(searches, xy_results): + results.append((s[0], r, [])) + self._logger.debug('--> results: {}'.format(results)) + return results + + def _country_code(self, country): + country_iso3166 = None + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + return country_iso3166 + diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index a8a9de0..ba6b853 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -39,17 +39,20 @@ class MapboxGeocoder(Traceable): def _parse_geocoder_response(self, response): json_response = json.loads(response) + self._logger.debug('--> json response: {}'.format(json_response)) - # If Mapbox returns more that one result, take the first one if json_response: - if type(json_response) == list: - json_response = json_response[0] + if type(json_response) != list: + json_response = [json_response] - if json_response[ENTRY_FEATURES]: - feature = json_response[ENTRY_FEATURES][0] - return self._extract_lng_lat_from_feature(feature) - else: - return [] + result = [] + for a_json_response in json_response: + if a_json_response[ENTRY_FEATURES]: + feature = a_json_response[ENTRY_FEATURES][0] + result.append(self._extract_lng_lat_from_feature(feature)) + else: + result.append([]) + return result else: return [] @@ -78,6 +81,13 @@ class MapboxGeocoder(Traceable): @qps_retry(qps=10) def geocode(self, searchtext, city=None, state_province=None, country=None): + """ + :param searchtext: + :param city: + :param state_province: + :param country: Country ISO 3166 code + :return: [x, y] on success, [] on error + """ if not self._validate_input(searchtext, city, state_province, country): return [] @@ -91,8 +101,21 @@ class MapboxGeocoder(Traceable): country = [country] if country else None + free_search = ', '.join(address) + + return self.geocode_free_text([free_search], country)[0] + + @qps_retry(qps=10) + def geocode_free_text(self, free_searches, country=None): + """ + :param free_searches: Free text searches + :param country: Country ISO 3166 code + :return: list of [x, y] on success, [] on error + """ try: - response = self._geocoder.forward(address=', '.join(address).decode('utf-8'), + free_search = ';'.join(free_searches) + self._logger.debug('--> free search: {}'.format(free_search)) + response = self._geocoder.forward(address=free_search.decode('utf-8'), country=country, limit=1) @@ -110,7 +133,7 @@ class MapboxGeocoder(Traceable): self._logger.error('Timeout connecting to Mapbox geocoding server', te) raise ServiceException('Error geocoding {0} using Mapbox'.format( - searchtext), None) + free_search), None) except requests.ConnectionError as ce: # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to Mapbox geocoding server', diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 2060385..2c5a6e9 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -81,10 +81,21 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 'Logroño, Spain': [-2.44998, 42.46592], }) + MAPBOX_POINTS = GOOGLE_POINTS.copy() + MAPBOX_POINTS.update({ + 'Logroño, Spain': [-2.44556, 42.47], + 'Logroño, Argentina': [-70.687195, -33.470901], # TODO: huge mismatch + 'Valladolid': [-4.72856, 41.652251], + 'Valladolid, Spain': [-4.72856, 41.652251], + '1902 amphitheatre parkway': [-118.03, 34.06], # TODO: huge mismatch + 'Madrid': [-3.69194, 40.4167754], + }) + FIXTURE_POINTS = { 'google': GOOGLE_POINTS, 'heremaps': HERE_POINTS, - 'tomtom': TOMTOM_POINTS + 'tomtom': TOMTOM_POINTS, + 'mapbox': MAPBOX_POINTS } def setUp(self): From 40ace9cfaa689644115438994718674fa1de24d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Jul 2018 17:42:08 +0200 Subject: [PATCH 102/265] Send _serial_geocode for single results --- .../cartodb_services/mapbox/bulk_geocoder.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py index cb27f22..5c502e3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -59,11 +59,7 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): def _batch_geocode(self, searches): if len(searches) == 1: - # A single search can take advantage of country fixing - search = searches[0] - elements = self._encoded_elements(search) - coordinates = self._geocode_search(*elements) - return [(search[0], coordinates, [])] + return self._serial_geocode(searches) else: frees = [] for search in searches: From cce5f92312b4afa3ad2a090a34b657882db72951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Jul 2018 18:17:45 +0200 Subject: [PATCH 103/265] Escape semicolons at Mapbox batch --- .../cartodb_services/mapbox/geocoder.py | 9 ++++++++- test/integration/test_street_functions.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index ba6b853..a3d6f93 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -113,7 +113,7 @@ class MapboxGeocoder(Traceable): :return: list of [x, y] on success, [] on error """ try: - free_search = ';'.join(free_searches) + free_search = ';'.join([self._escape(fs) for fs in free_searches]) self._logger.debug('--> free search: {}'.format(free_search)) response = self._geocoder.forward(address=free_search.decode('utf-8'), country=country, @@ -139,3 +139,10 @@ class MapboxGeocoder(Traceable): self._logger.error('Error connecting to Mapbox geocoding server', exception=ce) return [] + + def _escape(self, free_search): + # Semicolon is used to separate batch geocoding; there's no documented + # way to pass actual semicolons, and %3B or ; won't work (check + # TestBulkStreetFunctions.test_semicolon and the docs, + # https://www.mapbox.com/api-documentation/#batch-requests) + return free_search.replace(';', ',') diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 2c5a6e9..d1fdbd2 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -262,6 +262,19 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): response = self._run_authenticated(query) assert_equal(1, len(response['rows'])) + def test_semicolon(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1900 amphitheatre parkway; mountain view; ca; us\"}," \ + "{\"cartodb_id\": 2, \"address\": \"1900 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "'address', null, null, null)" + response = self._run_authenticated(query) + + x_y_by_cartodb_id = self._x_y_by_cartodb_id(response) + assert_equal(x_y_by_cartodb_id[1], x_y_by_cartodb_id[2]) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From 8ebd22bc26702aa5c21430f5c0e64f5e5a07810f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Jul 2018 14:09:42 +0200 Subject: [PATCH 104/265] Fixes error message check --- test/integration/test_street_functions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index d1fdbd2..12f5112 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -30,13 +30,14 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): assert_not_equal(geometry['geometry'], None) def test_if_select_with_street_without_api_key_raise_error(self): + table = self.env_variables['table_name'] query = "SELECT cdb_geocode_street_point(street) " \ - "as geometry FROM {0} LIMIT 1".format( - self.env_variables['table_name']) + "as geometry FROM {0} LIMIT 1".format(table) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_equal(e.message[0], + "permission denied for relation {}".format(table)) class TestBulkStreetFunctions(TestStreetFunctionsSetUp): From 8e430ce1c102f1911ea80caa6a67c28c55b18f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 9 Jul 2018 15:28:28 +0200 Subject: [PATCH 105/265] Google geocoder works better concatenating all components --- .../cartodb_services/geocoder.py | 4 ++++ .../cartodb_services/google/geocoder.py | 8 +++++--- test/integration/test_street_functions.py | 18 +++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 6a14594..6c2ce43 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -6,6 +6,10 @@ from collections import namedtuple import json +def compose_address(street, city=None, state=None, country=None): + return ', '.join(filter(None, [street, city, state, country])) + + def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches): plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 1cc61c9..956d09e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -6,6 +6,7 @@ from urlparse import parse_qs from exceptions import MalformedResult from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.geocoder import compose_address from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory @@ -36,8 +37,9 @@ class GoogleMapsGeocoder(StreetPointBulkGeocoder): def geocode(self, searchtext, city=None, state=None, country=None): try: + address = compose_address(searchtext, city, state, country) opt_params = self._build_optional_parameters(city, state, country) - results = self.geocoder.geocode(address=searchtext, + results = self.geocoder.geocode(address=address, components=opt_params) if results: return self._extract_lng_lat_from_result(results[0]) @@ -50,10 +52,10 @@ class GoogleMapsGeocoder(StreetPointBulkGeocoder): bulk_results = {} pool = Pool(processes=self.PARALLEL_PROCESSES) for search in searches: - (search_id, address, city, state, country) = search + (search_id, street, city, state, country) = search opt_params = self._build_optional_parameters(city, state, country) # Geocoding works better if components are also inside the address - address = ', '.join(filter(None, [address, city, state, country])) + address = compose_address(street, city, state, country) if address: self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) result = pool.apply_async(async_geocoder, diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 12f5112..d939de1 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -18,11 +18,18 @@ class TestStreetFunctionsSetUp(TestCase): self.env_variables['api_key'] ) + def _run_authenticated(self, query): + authenticated_query = "{}&api_key={}".format(query, + self.env_variables[ + 'api_key']) + return IntegrationTestHelper.execute_query_raw(self.sql_api_url, + authenticated_query) + class TestStreetFunctions(TestStreetFunctionsSetUp): def test_if_select_with_street_point_is_ok(self): - query = "SELECT cdb_geocode_street_point(street) " \ + query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( self.env_variables['table_name'], self.env_variables['api_key']) @@ -39,6 +46,15 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): assert_equal(e.message[0], "permission denied for relation {}".format(table)) + def test_component_aggregation(self): + query = "select st_x(the_geom), st_y(the_geom) from (" \ + "select cdb_dataservices_client.cdb_geocode_street_point( " \ + "'Plaza España 1', 'Barcelona', null, 'Spain') as the_geom) _x" + response = self._run_authenticated(query) + row = response['rows'][0] + x_y = [row['st_x'], row['st_y']] + # Wrong coordinates (Plaza España, Madrid): [-3.7138975, 40.4256762] + assert_close_enough(x_y, [2.1482563, 41.375485]) class TestBulkStreetFunctions(TestStreetFunctionsSetUp): provider = None From 1ffe3658fea0bb45734d78e44d0a4d666ae67815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 12:28:16 +0200 Subject: [PATCH 106/265] Revert "maxresults depends on batch" This reverts commit bf8b76b5fe4a079fd4405bfd615e493bca322e7f. --- .../cartodb_services/cartodb_services/here/bulk_geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index d652891..9a688f4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -24,7 +24,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): MIN_BATCHED_SEARCH = 100 # Under this, serial will be used JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed'] - def __init__(self, app_id, app_code, logger, service_params=None, maxresults=MAX_BATCH_SIZE): + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS): HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults) self.session = requests.Session() self.session.mount(self.BATCH_URL, From f6b7c13dde9010e54fc40959105b8e539edd46bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 13:25:40 +0200 Subject: [PATCH 107/265] GoogleMapsBulkGeocoder extraction --- .../cdb_dataservices_server--0.32.0.sql | 4 +- .../extension/sql/21_bulk_geocode_street.sql | 4 +- .../cartodb_services/google/__init__.py | 1 + .../cartodb_services/google/bulk_geocoder.py | 56 +++++++++++++++++++ .../cartodb_services/google/geocoder.py | 54 +----------------- test/integration/test_street_functions.py | 2 +- 6 files changed, 63 insertions(+), 58 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index 8543e7f..a013f3c 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -2387,10 +2387,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_stre RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services import run_street_point_geocoder from cartodb_services.tools import LegacyServiceManager - from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.google import GoogleMapsBulkGeocoder service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index adee498..309d323 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -44,10 +44,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_stre RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services import run_street_point_geocoder from cartodb_services.tools import LegacyServiceManager - from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.google import GoogleMapsBulkGeocoder service_manager = LegacyServiceManager('geocoder', username, orgname, GD) - geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py index 4e8cd6b..7570e31 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py @@ -1 +1,2 @@ from geocoder import GoogleMapsGeocoder +from bulk_geocoder import GoogleMapsBulkGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py new file mode 100644 index 0000000..aa174b8 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -0,0 +1,56 @@ +from multiprocessing import Pool +from exceptions import MalformedResult +from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.geocoder import compose_address +from cartodb_services.google import GoogleMapsGeocoder + + +def async_geocoder(geocoder, address, components): + results = geocoder.geocode(address=address, components=components) + return results if results else [] + + +class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): + """A Google Maps Geocoder wrapper for python""" + PARALLEL_PROCESSES = 13 + + def __init__(self, client_id, client_secret, logger): + GoogleMapsGeocoder.__init__(self, client_id, client_secret, logger) + + def _bulk_geocode(self, searches): + bulk_results = {} + pool = Pool(processes=self.PARALLEL_PROCESSES) + for search in searches: + (search_id, street, city, state, country) = search + opt_params = self._build_optional_parameters(city, state, country) + # Geocoding works better if components are also inside the address + address = compose_address(street, city, state, country) + if address: + self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) + result = pool.apply_async(async_geocoder, + (self.geocoder, address, opt_params)) + else: + result = [] + bulk_results[search_id] = result + pool.close() + pool.join() + + try: + results = [] + for search_id, bulk_result in bulk_results.items(): + try: + result = bulk_result.get() + except Exception as e: + self._logger.error('Error at Google async_geocoder', e) + result = [] + + lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] + results.append((search_id, lng_lat, [])) + return results + except KeyError as e: + self._logger.error('KeyError error', exception=e) + raise MalformedResult() + except Exception as e: + self._logger.error('General error', exception=e) + raise e + diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 956d09e..aa8877c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -1,30 +1,15 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- -import googlemaps from urlparse import parse_qs from exceptions import MalformedResult -from cartodb_services import StreetPointBulkGeocoder from cartodb_services.geocoder import compose_address from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory -from multiprocessing import Pool, TimeoutError -import time, random - -def async_geocoder(geocoder, address, components): - # TODO: clean this and previous import - # time.sleep(.3 + random.random()) - # return [{ 'geometry': { 'location': { 'lng': 1, 'lat': 2 } } }] - - results = geocoder.geocode(address=address, components=components) - return results if results else [] - -class GoogleMapsGeocoder(StreetPointBulkGeocoder): - """A Google Maps Geocoder wrapper for python""" - PARALLEL_PROCESSES = 13 +class GoogleMapsGeocoder(): def __init__(self, client_id, client_secret, logger): if client_id is None: @@ -48,43 +33,6 @@ class GoogleMapsGeocoder(StreetPointBulkGeocoder): except KeyError: raise MalformedResult() - def _bulk_geocode(self, searches): - bulk_results = {} - pool = Pool(processes=self.PARALLEL_PROCESSES) - for search in searches: - (search_id, street, city, state, country) = search - opt_params = self._build_optional_parameters(city, state, country) - # Geocoding works better if components are also inside the address - address = compose_address(street, city, state, country) - if address: - self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) - result = pool.apply_async(async_geocoder, - (self.geocoder, address, opt_params)) - else: - result = [] - bulk_results[search_id] = result - pool.close() - pool.join() - - try: - results = [] - for search_id, bulk_result in bulk_results.items(): - try: - result = bulk_result.get() - except Exception as e: - self._logger.error('Error at Google async_geocoder', e) - result = [] - - lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] - results.append((search_id, lng_lat, [])) - return results - except KeyError as e: - self._logger.error('KeyError error', exception=e) - raise MalformedResult() - except Exception as e: - self._logger.error('General error', exception=e) - raise e - def _extract_lng_lat_from_result(self, result): location = result['geometry']['location'] longitude = location['lng'] diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index d939de1..c35e1d5 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -38,7 +38,7 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): def test_if_select_with_street_without_api_key_raise_error(self): table = self.env_variables['table_name'] - query = "SELECT cdb_geocode_street_point(street) " \ + query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1".format(table) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) From a6c5c211312d4a5d2d60db27ba3f6f81c6e9fe4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 13:45:01 +0200 Subject: [PATCH 108/265] Serial geocode for Google bulk --- .../cartodb_services/google/bulk_geocoder.py | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index aa174b8..206fa54 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -12,32 +12,59 @@ def async_geocoder(geocoder, address, components): class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): """A Google Maps Geocoder wrapper for python""" + MAX_BATCH_SIZE = 1000 + MIN_BATCHED_SEARCH = 2 # Batched is a parallelization PARALLEL_PROCESSES = 13 def __init__(self, client_id, client_secret, logger): GoogleMapsGeocoder.__init__(self, client_id, client_secret, logger) def _bulk_geocode(self, searches): + if len(searches) > self.MAX_BATCH_SIZE: + raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) + if self._should_use_batch(searches): + self._logger.debug('--> Batch geocode') + return self._batch_geocode(searches) + else: + self._logger.debug('--> Serial geocode') + return self._serial_geocode(searches) + + def _should_use_batch(self, searches): + return len(searches) >= self.MIN_BATCHED_SEARCH + + def _serial_geocode(self, searches): + results = [] + for search in searches: + (cartodb_id, street, city, state, country) = search + address = compose_address(street, city, state, country) + components = self._build_optional_parameters(city, state, country) + result = self.geocoder.geocode(address=address, components=components) + lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] + self._logger.debug('--> lng_lat: {}'.format(lng_lat)) + results.append((cartodb_id, lng_lat, [])) + return results + + def _batch_geocode(self, searches): bulk_results = {} pool = Pool(processes=self.PARALLEL_PROCESSES) for search in searches: - (search_id, street, city, state, country) = search - opt_params = self._build_optional_parameters(city, state, country) + (cartodb_id, street, city, state, country) = search + components = self._build_optional_parameters(city, state, country) # Geocoding works better if components are also inside the address address = compose_address(street, city, state, country) if address: - self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), opt_params)) + self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), components)) result = pool.apply_async(async_geocoder, - (self.geocoder, address, opt_params)) + (self.geocoder, address, components)) else: result = [] - bulk_results[search_id] = result + bulk_results[cartodb_id] = result pool.close() pool.join() try: results = [] - for search_id, bulk_result in bulk_results.items(): + for cartodb_id, bulk_result in bulk_results.items(): try: result = bulk_result.get() except Exception as e: @@ -45,7 +72,7 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): result = [] lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] - results.append((search_id, lng_lat, [])) + results.append((cartodb_id, lng_lat, [])) return results except KeyError as e: self._logger.error('KeyError error', exception=e) From 286a75fa8e43f64b7db57083ba82c22e997ee0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 15:17:14 +0200 Subject: [PATCH 109/265] _bulk_geocode logic extraction --- client/cdb_dataservices_client--0.25.0.sql | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- .../cartodb_services/geocoder.py | 31 +++- .../cartodb_services/google/bulk_geocoder.py | 10 -- .../cartodb_services/here/bulk_geocoder.py | 14 +- .../cartodb_services/mapbox/bulk_geocoder.py | 10 -- .../cartodb_services/tomtom/bulk_geocoder.py | 14 +- test/integration/test_street_functions.py | 155 ++++++++++-------- 8 files changed, 115 insertions(+), 123 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index e810dbc..d21bb5f 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1987,7 +1987,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 100) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 50) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index d0bcf5e..2e14965 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 100) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 50) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 6c2ce43..c25216c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -51,7 +51,12 @@ StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state class StreetPointBulkGeocoder: """ Classes extending StreetPointBulkGeocoder should implement: - * _bulk_geocode(street_geocoder_searches) + * _batch_geocode(street_geocoder_searches) + * MAX_BATCH_SIZE + + If they want to provide an alternative serial (for small batches): + * _should_use_batch(street_geocoder_searches) + * _serial_geocode(street_geocoder_searches) """ SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] @@ -77,10 +82,22 @@ class StreetPointBulkGeocoder: street_geocoder_searches.append( StreetGeocoderSearch(search_id, address, city, state, country)) - return self._bulk_geocode(street_geocoder_searches) + if len(street_geocoder_searches) > self.MAX_BATCH_SIZE: + raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) + if self._should_use_batch(street_geocoder_searches): + self._logger.debug('--> Batch geocode') + return self._batch_geocode(street_geocoder_searches) + else: + self._logger.debug('--> Serial geocode') + return self._serial_geocode(street_geocoder_searches) + + def _batch_geocode(self, street_geocoder_searches): + raise NotImplementedError('Subclasses must implement _batch_geocode') + + def _serial_geocode(self, street_geocoder_searches): + raise NotImplementedError('Subclasses must implement _serial_geocode') + + def _should_use_batch(self, street_geocoder_searches): + return True + - def _bulk_geocode(self, street_geocoder_searches): - """ - Subclasses must implement _bulk_geocode - """ - raise NotImplementedError('Subclasses must implement _bulk_geocode') diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index 206fa54..e59e408 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -19,16 +19,6 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): def __init__(self, client_id, client_secret, logger): GoogleMapsGeocoder.__init__(self, client_id, client_secret, logger) - def _bulk_geocode(self, searches): - if len(searches) > self.MAX_BATCH_SIZE: - raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) - if self._should_use_batch(searches): - self._logger.debug('--> Batch geocode') - return self._batch_geocode(searches) - else: - self._logger.debug('--> Serial geocode') - return self._serial_geocode(searches) - def _should_use_batch(self, searches): return len(searches) >= self.MIN_BATCHED_SEARCH diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 9a688f4..e567e2e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -15,13 +15,13 @@ from cartodb_services.tools.exceptions import ServiceException HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status') class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): - BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' MAX_BATCH_SIZE = 1000000 # From the docs + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] MAX_STALLED_RETRIES = 100 BATCH_RETRY_SLEEP_S = 5 - MIN_BATCHED_SEARCH = 100 # Under this, serial will be used JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed'] def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS): @@ -34,16 +34,6 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): 'app_code': self.app_code, } - def _bulk_geocode(self, searches): - if len(searches) > self.MAX_BATCH_SIZE: - raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) - if self._should_use_batch(searches): - self._logger.debug('--> Batch geocode') - return self._batch_geocode(searches) - else: - self._logger.debug('--> Serial geocode') - return self._serial_geocode(searches) - def _should_use_batch(self, searches): return len(searches) >= self.MIN_BATCHED_SEARCH diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py index 5c502e3..63f5e72 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -21,16 +21,6 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) self.session = requests.Session() - def _bulk_geocode(self, searches): - if len(searches) > self.MAX_BATCH_SIZE: - raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) - if self._should_use_batch(searches): - self._logger.debug('--> Batch geocode') - return self._batch_geocode(searches) - else: - self._logger.debug('--> Serial geocode') - return self._serial_geocode(searches) - def _should_use_batch(self, searches): return len(searches) >= self.MIN_BATCHED_SEARCH diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index 8036667..a3d8583 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -6,12 +6,12 @@ from cartodb_services.tools.exceptions import ServiceException class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): + MAX_BATCH_SIZE = 1000000 # From the docs + MIN_BATCHED_SEARCH = 10 # Batch API is really fast BASE_URL = 'https://api.tomtom.com' BATCH_URL = BASE_URL + '/search/2/batch.json' - MAX_BATCH_SIZE = 1000000 # From the docs MAX_STALLED_RETRIES = 100 BATCH_RETRY_SLEEP_S = 5 - MIN_BATCHED_SEARCH = 10 # Batch API is really fast READ_TIMEOUT = 60 CONNECT_TIMEOUT = 10 MAX_RETRIES = 1 @@ -26,16 +26,6 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): self.session.mount(self.BATCH_URL, HTTPAdapter(max_retries=self.max_retries)) - def _bulk_geocode(self, searches): - if len(searches) > self.MAX_BATCH_SIZE: - raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) - if self._should_use_batch(searches): - self._logger.debug('--> Batch geocode') - return self._batch_geocode(searches) - else: - self._logger.debug('--> Serial geocode') - return self._serial_geocode(searches) - def _should_use_batch(self, searches): return len(searches) >= self.MIN_BATCHED_SEARCH diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index c35e1d5..d66df30 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -8,6 +8,67 @@ from ..helpers.integration_test_helper import IntegrationTestHelper from ..helpers.integration_test_helper import assert_close_enough class TestStreetFunctionsSetUp(TestCase): + provider = None + fixture_points = None + + GOOGLE_POINTS = { + 'Plaza Mayor, Valladolid': [-4.728252, 41.6517025], + 'Paseo Zorrilla, Valladolid': [-4.7404453, 41.6314339], + '1900 amphitheatre parkway': [-122.0875324, 37.4227968], + '1901 amphitheatre parkway': [-122.0885504, 37.4238657], + '1902 amphitheatre parkway': [-122.0876674, 37.4235729], + 'Valladolid': [-4.7245321, 41.652251], + 'Valladolid, Spain': [-4.7245321, 41.652251], + 'Valladolid, Mexico': [-88.2022488, 20.68964], + 'Madrid': [-3.7037902, 40.4167754], + 'Logroño, Spain': [-2.4449852, 42.4627195], + 'Logroño, Argentina': [-61.6961807, -29.5031057], + 'Plaza España 1, Barcelona': [2.1482563, 41.375485] + } + + HERE_POINTS = { + 'Plaza Mayor, Valladolid': [-4.72979, 41.65258], + 'Paseo Zorrilla, Valladolid': [-4.73869, 41.63817], + '1900 amphitheatre parkway': [-122.0879468, 37.4234763], + '1901 amphitheatre parkway': [-122.0879253, 37.4238725], + '1902 amphitheatre parkway': [-122.0879531, 37.4234775], + 'Valladolid': [-4.73214, 41.6542], + 'Valladolid, Spain': [-4.73214, 41.6542], + 'Valladolid, Mexico': [-88.20117, 20.69021], + 'Madrid': [-3.70578, 40.42028], + 'Logroño, Spain': [-2.45194, 42.46592], + 'Logroño, Argentina': [-61.69604, -29.50425], + 'Plaza España 1, Barcelona': [2.1735699, 41.3823] # TODO: not ideal + } + + TOMTOM_POINTS = HERE_POINTS.copy() + TOMTOM_POINTS.update({ + 'Plaza Mayor, Valladolid': [-4.72183, 41.5826], + 'Paseo Zorrilla, Valladolid': [-4.74031, 41.63181], + 'Valladolid': [-4.72838, 41.6542], + 'Valladolid, Spain': [-4.72838, 41.6542], + 'Madrid': [-3.70035, 40.42028], + 'Logroño, Spain': [-2.44998, 42.46592], + 'Plaza España 1, Barcelona': [2.07479, 41.36818] # TODO: not ideal + }) + + MAPBOX_POINTS = GOOGLE_POINTS.copy() + MAPBOX_POINTS.update({ + 'Logroño, Spain': [-2.44556, 42.47], + 'Logroño, Argentina': [-70.687195, -33.470901], # TODO: huge mismatch + 'Valladolid': [-4.72856, 41.652251], + 'Valladolid, Spain': [-4.72856, 41.652251], + '1902 amphitheatre parkway': [-118.03, 34.06], # TODO: huge mismatch + 'Madrid': [-3.69194, 40.4167754], + 'Plaza España 1, Barcelona': [2.245969, 41.452483] # TODO: not ideal + }) + + FIXTURE_POINTS = { + 'google': GOOGLE_POINTS, + 'heremaps': HERE_POINTS, + 'tomtom': TOMTOM_POINTS, + 'mapbox': MAPBOX_POINTS + } def setUp(self): self.env_variables = IntegrationTestHelper.get_environment_variables() @@ -18,6 +79,15 @@ class TestStreetFunctionsSetUp(TestCase): self.env_variables['api_key'] ) + if not self.fixture_points: + query = "select provider from " \ + "cdb_dataservices_client.cdb_service_quota_info() " \ + "where service = 'hires_geocoder'" + response = self._run_authenticated(query) + provider = response['rows'][0]['provider'] + self.fixture_points = self.FIXTURE_POINTS[provider] + + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ @@ -54,77 +124,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): row = response['rows'][0] x_y = [row['st_x'], row['st_y']] # Wrong coordinates (Plaza España, Madrid): [-3.7138975, 40.4256762] - assert_close_enough(x_y, [2.1482563, 41.375485]) + assert_close_enough(x_y, self.fixture_points['Plaza España 1, Barcelona']) class TestBulkStreetFunctions(TestStreetFunctionsSetUp): - provider = None - fixture_points = None - - GOOGLE_POINTS = { - 'Plaza Mayor, Valladolid': [-4.728252, 41.6517025], - 'Paseo Zorrilla, Valladolid': [-4.7404453, 41.6314339], - '1900 amphitheatre parkway': [-122.0875324, 37.4227968], - '1901 amphitheatre parkway': [-122.0885504, 37.4238657], - '1902 amphitheatre parkway': [-122.0876674, 37.4235729], - 'Valladolid': [-4.7245321, 41.652251], - 'Valladolid, Spain': [-4.7245321, 41.652251], - 'Valladolid, Mexico': [-88.2022488, 20.68964], - 'Madrid': [-3.7037902, 40.4167754], - 'Logroño, Spain': [-2.4449852, 42.4627195], - 'Logroño, Argentina': [-61.6961807, -29.5031057] - } - - HERE_POINTS = { - 'Plaza Mayor, Valladolid': [-4.72979, 41.65258], - 'Paseo Zorrilla, Valladolid': [-4.73869, 41.63817], - '1900 amphitheatre parkway': [-122.0879468, 37.4234763], - '1901 amphitheatre parkway': [-122.0879253, 37.4238725], - '1902 amphitheatre parkway': [-122.0879531, 37.4234775], - 'Valladolid': [-4.73214, 41.6542], - 'Valladolid, Spain': [-4.73214, 41.6542], - 'Valladolid, Mexico': [-88.20117, 20.69021], - 'Madrid': [-3.70578, 40.42028], - 'Logroño, Spain': [-2.45194, 42.46592], - 'Logroño, Argentina': [-61.69604, -29.50425] - } - - TOMTOM_POINTS = HERE_POINTS.copy() - TOMTOM_POINTS.update({ - 'Plaza Mayor, Valladolid': [-4.72183, 41.5826], - 'Paseo Zorrilla, Valladolid': [-4.74031, 41.63181], - 'Valladolid': [-4.72838, 41.6542], - 'Valladolid, Spain': [-4.72838, 41.6542], - 'Madrid': [-3.70035, 40.42028], - 'Logroño, Spain': [-2.44998, 42.46592], - }) - - MAPBOX_POINTS = GOOGLE_POINTS.copy() - MAPBOX_POINTS.update({ - 'Logroño, Spain': [-2.44556, 42.47], - 'Logroño, Argentina': [-70.687195, -33.470901], # TODO: huge mismatch - 'Valladolid': [-4.72856, 41.652251], - 'Valladolid, Spain': [-4.72856, 41.652251], - '1902 amphitheatre parkway': [-118.03, 34.06], # TODO: huge mismatch - 'Madrid': [-3.69194, 40.4167754], - }) - - FIXTURE_POINTS = { - 'google': GOOGLE_POINTS, - 'heremaps': HERE_POINTS, - 'tomtom': TOMTOM_POINTS, - 'mapbox': MAPBOX_POINTS - } - - def setUp(self): - TestStreetFunctionsSetUp.setUp(self) - - if not self.fixture_points: - query = "select provider from " \ - "cdb_dataservices_client.cdb_service_quota_info() " \ - "where service = 'hires_geocoder'" - response = self._run_authenticated(query) - provider = response['rows'][0]['provider'] - self.fixture_points = self.FIXTURE_POINTS[provider] def test_full_spec(self): query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ @@ -257,7 +259,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): """ Useful just to test a good batch size """ - n = 10 + n = 50 streets = [] for i in range(0, n): streets.append('{{"cartodb_id": {}, "address": "{} Yonge Street, ' \ @@ -292,6 +294,19 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): x_y_by_cartodb_id = self._x_y_by_cartodb_id(response) assert_equal(x_y_by_cartodb_id[1], x_y_by_cartodb_id[2]) + # "'Plaza España 1', 'Barcelona', null, 'Spain') as the_geom) _x" + def test_component_aggregation(self): + query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 1 as cartodb_id, ''Spain'' as country, " \ + "''Barcelona'' as city, " \ + "''Plaza España 1'' as street' " \ + ", 'street', 'city', NULL, 'country')" + response = self._run_authenticated(query) + + assert_close_enough(self._x_y_by_cartodb_id(response)[1], + self.fixture_points['Plaza España 1, Barcelona']) + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From 531ad281589eaa838a76cd6fcd1d18c606b200d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 19:06:49 +0200 Subject: [PATCH 110/265] Send optimal batch size --- client/cdb_dataservices_client--0.25.0.sql | 64 +++++++++++-------- client/sql/16_custom_types.sql | 3 +- client/sql/21_bulk_geocoding_functions.sql | 61 ++++++++++-------- .../21_bulk_geocoding_functions_test.out | 20 +++--- .../sql/21_bulk_geocoding_functions_test.sql | 18 +++--- .../cdb_dataservices_server--0.32.0.sql | 17 +++-- server/extension/sql/200_quotas.sql | 17 +++-- .../cartodb_services/bulk_geocoders.py | 11 ++++ .../cartodb_services/here/bulk_geocoder.py | 13 ++-- test/integration/test_street_functions.py | 19 +++++- 10 files changed, 153 insertions(+), 90 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index d21bb5f..2bed351 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -112,7 +112,8 @@ CREATE TYPE cdb_dataservices_client.service_quota_info AS ( monthly_quota NUMERIC, used_quota NUMERIC, soft_limit BOOLEAN, - provider TEXT + provider TEXT, + max_batch_size NUMERIC ); -- -- Public dataservices API function @@ -1987,25 +1988,36 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( TARGET cdb_dataservices_server._DST_DisconnectUserTable; $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 50) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; remaining_quota integer; + max_batch_size integer; cartodb_id_batch integer; batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; - MAX_BATCH_SIZE CONSTANT numeric := 10000; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; current_row_count integer ; temp_table_name text; BEGIN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + IF batch_size IS NULL THEN - RAISE EXCEPTION 'batch_size can''t be null'; - ELSIF batch_size > MAX_BATCH_SIZE THEN - RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; END IF; EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; @@ -2013,11 +2025,7 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF enough_quota IS NOT NULL AND NOT enough_quota THEN - SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota - INTO remaining_quota - FROM cdb_dataservices_client.cdb_service_quota_info() csqi - WHERE service = 'hires_geocoder'; + IF remaining_quota < query_row_count THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; @@ -2036,25 +2044,27 @@ BEGIN coalesce(state_column, ''''''), coalesce(country_column, '''''') into street_column, city_column, state_column, country_column; - FOR cartodb_id_batch in 0..(batches_n - 1) - LOOP + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP - EXECUTE format( - 'WITH geocoding_data as (' || - ' SELECT ' || - ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id-1)::float/$1) as batch' || - ' FROM (%s) _x' || - ') ' || - 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || - 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING batch_size, cartodb_id_batch; + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((cartodb_id-1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; + GET DIAGNOSTICS current_row_count = ROW_COUNT; + RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; + END LOOP; + END IF; RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); END; diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 1b74cac..7458ffa 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -39,5 +39,6 @@ CREATE TYPE cdb_dataservices_client.service_quota_info AS ( monthly_quota NUMERIC, used_quota NUMERIC, soft_limit BOOLEAN, - provider TEXT + provider TEXT, + max_batch_size NUMERIC ); diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 2e14965..f9ba824 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,23 +1,34 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, - street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT 50) + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; remaining_quota integer; + max_batch_size integer; cartodb_id_batch integer; batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; - MAX_BATCH_SIZE CONSTANT numeric := 10000; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; current_row_count integer ; temp_table_name text; BEGIN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + IF batch_size IS NULL THEN - RAISE EXCEPTION 'batch_size can''t be null'; - ELSIF batch_size > MAX_BATCH_SIZE THEN - RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; END IF; EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; @@ -25,11 +36,7 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF enough_quota IS NOT NULL AND NOT enough_quota THEN - SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota - INTO remaining_quota - FROM cdb_dataservices_client.cdb_service_quota_info() csqi - WHERE service = 'hires_geocoder'; + IF remaining_quota < query_row_count THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; @@ -48,25 +55,27 @@ BEGIN coalesce(state_column, ''''''), coalesce(country_column, '''''') into street_column, city_column, state_column, country_column; - FOR cartodb_id_batch in 0..(batches_n - 1) - LOOP + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP - EXECUTE format( - 'WITH geocoding_data as (' || - ' SELECT ' || - ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id-1)::float/$1) as batch' || - ' FROM (%s) _x' || - ') ' || - 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || - 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING batch_size, cartodb_id_batch; + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((cartodb_id-1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; + GET DIAGNOSTICS current_row_count = ROW_COUNT; + RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; + END LOOP; + END IF; RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); END; diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index 7ca70db..d2f268b 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -1,21 +1,21 @@ \set VERBOSITY terse --- Test bulk size mandatory -SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -ERROR: batch_size can't be null --- Test quota check by mocking quota 0 +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; +$$ LANGUAGE SQL; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ - SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; -$$ LANGUAGE SQL; +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ERROR: Remaining quota: 0. Estimated cost: 1 +-- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Remaining quota: 0. Estimated cost: 1 DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index 5433bd7..d17f470 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -1,26 +1,26 @@ \set VERBOSITY terse --- Test bulk size mandatory -SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; +$$ LANGUAGE SQL; --- Test quota check by mocking quota 0 ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ - SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; -$$ LANGUAGE SQL; +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +-- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index a013f3c..dc82322 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -1861,7 +1861,8 @@ BEGIN monthly_quota NUMERIC, used_quota NUMERIC, soft_limit BOOLEAN, - provider TEXT + provider TEXT, + max_batch_size NUMERIC ); END IF; END $$; @@ -1872,6 +1873,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -1889,7 +1891,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] #-- Hires Geocoder service = 'hires_geocoder' @@ -1901,7 +1903,12 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] #-- Routing service = 'routing' @@ -1913,7 +1920,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] #-- Observatory service = 'observatory' @@ -1925,7 +1932,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 28ee724..6f34083 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -22,7 +22,8 @@ BEGIN monthly_quota NUMERIC, used_quota NUMERIC, soft_limit BOOLEAN, - provider TEXT + provider TEXT, + max_batch_size NUMERIC ); END IF; END $$; @@ -33,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -50,7 +52,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] #-- Hires Geocoder service = 'hires_geocoder' @@ -62,7 +64,12 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] #-- Routing service = 'routing' @@ -74,7 +81,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] #-- Observatory service = 'observatory' @@ -86,7 +93,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py new file mode 100644 index 0000000..6dfd555 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py @@ -0,0 +1,11 @@ +from google import GoogleMapsBulkGeocoder +from here import HereMapsBulkGeocoder +from tomtom import TomTomBulkGeocoder +from mapbox import MapboxBulkGeocoder + +BATCH_GEOCODER_CLASS_BY_PROVIDER = { + 'google': GoogleMapsBulkGeocoder, + 'heremaps': HereMapsBulkGeocoder, + 'tomtom': TomTomBulkGeocoder, + 'mapbox': MapboxBulkGeocoder +} diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index e567e2e..3724bcd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -16,7 +16,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status' class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): MAX_BATCH_SIZE = 1000000 # From the docs - MIN_BATCHED_SEARCH = 100 # Under this, serial will be used + MIN_BATCHED_SEARCH = 1000 # Under this, serial will be used BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] @@ -55,14 +55,17 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): while True: job_info = self._job_status(request_id) if job_info.processed_count == last_processed: + self._logger.debug('--> no progress ({})'.format(last_processed)) stalled_retries += 1 if stalled_retries > self.MAX_STALLED_RETRIES: raise Exception('Too many retries for job {}'.format(request_id)) else: + self._logger.debug('--> progress ({} != {})'.format(job_info.processed_count, last_processed)) stalled_retries = 0 last_processed = job_info.processed_count - self._logger.debug('--> Job poll check: {}'.format(job_info)) + self._logger.debug('--> Job poll check ({}): {}'.format( + stalled_retries, job_info)) if job_info.status in self.JOB_FINAL_STATES: break else: @@ -95,7 +98,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): request_params.update({ 'gen': 8, 'action': 'run', - #'mailto': 'juanignaciosl@carto.com', + # 'mailto': 'juanignaciosl@carto.com', 'header': 'true', 'inDelim': '|', 'outDelim': '|', @@ -121,8 +124,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): timeout=(self.connect_timeout, self.read_timeout)) polling_root = ET.fromstring(polling_r.text) return HereJobStatus( - total_count=polling_root.find('./Response/TotalCount').text, - processed_count=polling_root.find('./Response/ProcessedCount').text, + total_count=int(polling_root.find('./Response/TotalCount').text), + processed_count=int(polling_root.find('./Response/ProcessedCount').text), status=polling_root.find('./Response/Status').text) def _download_results(self, job_id): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index d66df30..52ffbe0 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -259,7 +259,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): """ Useful just to test a good batch size """ - n = 50 + n = 110 + batch_size = 'NULL' # NULL for optimal streets = [] for i in range(0, n): streets.append('{{"cartodb_id": {}, "address": "{} Yonge Street, ' \ @@ -270,7 +271,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "'select * from jsonb_to_recordset(''[" \ "{}" \ "]''::jsonb) as (cartodb_id integer, address text)', " \ - "'address', null, null, null, {})".format(','.join(streets), n) + "'address', null, null, null, {})".format(','.join(streets), batch_size) response = self._run_authenticated(query) assert_equal(n - 1, len(response['rows'])) @@ -307,6 +308,20 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_close_enough(self._x_y_by_cartodb_id(response)[1], self.fixture_points['Plaza España 1, Barcelona']) + def _test_known_table(self): + subquery = 'select * from known_table where cartodb_id < 1100' + subquery_count = 'select count(1) from ({}) _x'.format(subquery) + count = self._run_authenticated(subquery_count)['rows'][0]['count'] + + query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'{}' " \ + ", 'street', 'city', NULL, 'country')".format(subquery) + response = self._run_authenticated(query) + assert_equal(len(response['rows']), count) + assert_not_equal(response['rows'][0]['st_x'], None) + + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ From 34e622b8092b8e37ab8a6d9d5856126a529ffe69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 20:30:01 +0200 Subject: [PATCH 111/265] Relevance metadata for HERE --- .../cartodb_services/geocoder.py | 10 ++++++++-- .../cartodb_services/here/bulk_geocoder.py | 12 ++++++++---- .../cartodb_services/here/geocoder.py | 17 +++++++++++++---- test/integration/test_street_functions.py | 15 ++++++++++++--- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index c25216c..cea8ef4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -22,12 +22,18 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org if geocode_results: results = [] for result in geocode_results: + if len(result) > 2: + metadata = json.dumps(result[2]) + else: + logger.warning('Geocoding for {} without metadata'.format(username)) + metadata = '{}' + if result[1] and len(result[1]) == 2: plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], None]) + results.append([result[0], point['the_geom'], metadata]) else: - results.append([result[0], None, None]) + results.append([result[0], None, metadata]) service_manager.quota_service.increment_success_service_use(len(results)) return results else: diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 3724bcd..dd47fda 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -16,7 +16,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status' class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): MAX_BATCH_SIZE = 1000000 # From the docs - MIN_BATCHED_SEARCH = 1000 # Under this, serial will be used + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] @@ -41,8 +41,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (search_id, address, city, state, country) = search - coordinates = self.geocode(searchtext=address, city=city, state=state, country=country) - results.append((search_id, coordinates, [])) + result = self.geocode_meta(searchtext=address, city=city, state=state, country=country) + results.append((search_id, result[0], result[1])) return results def _batch_geocode(self, searches): @@ -140,7 +140,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): reader = csv.DictReader(root_zip.open(name), delimiter='|') for row in reader: if row['SeqNumber'] == '1': # First per requested data - results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']])) + results.append((row['recId'], + [row['displayLongitude'], row['displayLatitude']], + { + 'relevance': float(row['relevance']) + })) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 6c3b81c..8139e1a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -65,12 +65,15 @@ class HereMapsGeocoder(Traceable): self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) def geocode(self, **kwargs): + return self.geocode_meta(**kwargs)[0] + + def geocode_meta(self, **kwargs): params = {} for key, value in kwargs.iteritems(): if value and value.strip(): params[key] = value if not params: - return [] + return [[], {}] return self._execute_geocode(params) def _execute_geocode(self, params): @@ -78,10 +81,11 @@ class HereMapsGeocoder(Traceable): raise BadGeocodingParams(params) try: response = self._perform_request(params) - results = response['Response']['View'][0]['Result'][0] - return self._extract_lng_lat_from_result(results) + result = response['Response']['View'][0]['Result'][0] + return [self._extract_lng_lat_from_result(result), + self._extract_metadata_from_result(result)] except IndexError: - return [] + return [[], {}] except KeyError: raise MalformedResult() @@ -118,3 +122,8 @@ class HereMapsGeocoder(Traceable): latitude = location['DisplayPosition']['Latitude'] return [longitude, latitude] + + def _extract_metadata_from_result(self, result): + return { + 'relevance': result['Relevance'] + } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 52ffbe0..e842396 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -2,10 +2,9 @@ # -*- coding: utf-8 -*- from unittest import TestCase -from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper -from ..helpers.integration_test_helper import assert_close_enough +from ..helpers.integration_test_helper import assert_close_enough, isclose class TestStreetFunctionsSetUp(TestCase): provider = None @@ -321,6 +320,16 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(len(response['rows']), count) assert_not_equal(response['rows'][0]['st_x'], None) + def test_relevance(self): + query = "select metadata " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 1 as cartodb_id, ''Spain'' as country, " \ + "''Barcelona'' as city, " \ + "''Plaza España 1'' as street' " \ + ", 'street', 'city', NULL, 'country')" + response = self._run_authenticated(query) + + assert_true(isclose(response['rows'][0]['metadata']['relevance'], 1)) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From 2af92045427318f1b6d6463a21b93043996b033b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 10 Jul 2018 21:21:42 +0200 Subject: [PATCH 112/265] Relevance metadata for TomTom --- .../cartodb_services/tomtom/bulk_geocoder.py | 10 ++++---- .../cartodb_services/tomtom/geocoder.py | 25 ++++++++++++++----- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index a3d8583..c8ad803 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -38,18 +38,18 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): state = state.encode('utf-8') if state else None country = country.encode('utf-8') if country else None self._logger.debug('--> Sending serial search: {}'.format(search)) - coordinates = self.geocode(searchtext=address, city=city, + result = self.geocode_meta(searchtext=address, city=city, state_province=state, country=country) self._logger.debug('--> result sent') - results.append((search_id, coordinates, [])) + results.append((search_id, result[0], result[1])) return results def _batch_geocode(self, searches): location = self._send_batch(searches) - xy_results = self._download_results(location) + full_results = self._download_results(location) results = [] - for s, r in zip(searches, xy_results): - results.append((s[0], r, [])) + for s, r in zip(searches, full_results): + results.append((s[0], r[0], r[1])) self._logger.debug('--> results: {}'.format(results)) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index b4e82cb..b064ea5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -18,6 +18,7 @@ ENTRY_RESULTS = 'results' ENTRY_POSITION = 'position' ENTRY_LON = 'lon' ENTRY_LAT = 'lat' +EMPTY_RESPONSE = [[], {}] class TomTomGeocoder(Traceable): @@ -62,6 +63,11 @@ class TomTomGeocoder(Traceable): @qps_retry(qps=5) def geocode(self, searchtext, city=None, state_province=None, country=None): + return self.geocode_meta(searchtext, city, state_province, country)[0] + + @qps_retry(qps=5) + def geocode_meta(self, searchtext, city=None, state_province=None, + country=None): if searchtext: searchtext = searchtext.decode('utf-8') if city: @@ -72,7 +78,7 @@ class TomTomGeocoder(Traceable): country = country.decode('utf-8') if not self._validate_input(searchtext, city, state_province, country): - return [] + return EMPTY_RESPONSE address = [] if searchtext and searchtext.strip(): @@ -98,15 +104,15 @@ class TomTomGeocoder(Traceable): # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to TomTom geocoding server', exception=ce) - return [] + return EMPTY_RESPONSE def _parse_response(self, status_code, text): if status_code == requests.codes.ok: return self._parse_geocoder_response(text) elif status_code == requests.codes.bad_request: - return [] + return EMPTY_RESPONSE elif status_code == requests.codes.unprocessable_entity: - return [] + return EMPTY_RESPONSE else: msg = 'Unknown response {}: {}'.format(str(status_code), text) raise ServiceException(msg, None) @@ -117,7 +123,14 @@ class TomTomGeocoder(Traceable): if json_response and json_response[ENTRY_RESULTS]: result = json_response[ENTRY_RESULTS][0] - return self._extract_lng_lat_from_feature(result) + return [ + self._extract_lng_lat_from_feature(result), + self._extract_metadata_from_result(result) + ] else: - return [] + return EMPTY_RESPONSE + def _extract_metadata_from_result(self, result): + return { + 'relevance': result['score'] # TODO: normalize + } From 825e3b7ee82080b4998849d85bb7a76d8e1c2ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 07:42:21 +0200 Subject: [PATCH 113/265] Relevance metadata for Mapbox --- .../cartodb_services/mapbox/bulk_geocoder.py | 19 +++----- .../cartodb_services/mapbox/geocoder.py | 46 +++++++++++++------ test/integration/test_street_functions.py | 40 +++++++++++----- 3 files changed, 68 insertions(+), 37 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py index 63f5e72..3ddd457 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -29,15 +29,10 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): for search in searches: elements = self._encoded_elements(search) self._logger.debug('--> Sending serial search: {}'.format(search)) - coordinates = self._geocode_search(*elements) - results.append((search[0], coordinates, [])) - return results + result = self.geocode_meta(*elements) - def _geocode_search(self, address, city, state, country): - coordinates = self.geocode(searchtext=address, city=city, - state_province=state, country=country) - self._logger.debug('--> result sent') - return coordinates + results.append((search[0], result[0], result[1])) + return results def _encoded_elements(self, search): (search_id, address, city, state, country) = search @@ -58,11 +53,11 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): frees.append(free) self._logger.debug('--> sending free search: {}'.format(frees)) - xy_results = self.geocode_free_text(frees) + full_results = self.geocode_free_text_meta(frees) results = [] - self._logger.debug('--> searches: {}; xy: {}'.format(searches, xy_results)) - for s, r in zip(searches, xy_results): - results.append((s[0], r, [])) + self._logger.debug('--> searches: {}; xy: {}'.format(searches, full_results)) + for s, r in zip(searches, full_results): + results.append((s[0], r[0], r[1])) self._logger.debug('--> results: {}'.format(results)) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index a3d6f93..5d644d9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -22,6 +22,8 @@ ENTRY_COORDINATES = 'coordinates' ENTRY_TYPE = 'type' TYPE_POINT = 'Point' +EMPTY_RESPONSE = [[], {}] + class MapboxGeocoder(Traceable): ''' @@ -49,12 +51,16 @@ class MapboxGeocoder(Traceable): for a_json_response in json_response: if a_json_response[ENTRY_FEATURES]: feature = a_json_response[ENTRY_FEATURES][0] - result.append(self._extract_lng_lat_from_feature(feature)) + result.append([ + self._extract_lng_lat_from_feature(feature), + self._extract_metadata_from_result(feature) + ] + ) else: - result.append([]) + result.append(EMPTY_RESPONSE) return result else: - return [] + return EMPTY_RESPONSE def _extract_lng_lat_from_feature(self, feature): geometry = feature[ENTRY_GEOMETRY] @@ -67,6 +73,14 @@ class MapboxGeocoder(Traceable): latitude = location[1] return [longitude, latitude] + def _extract_metadata_from_result(self, result): + return { + 'relevance': self._normalize_relevance(float(result['relevance'])) + } + + def _normalize_relevance(self, relevance): + return 1 if relevance == 0.99 else relevance + def _validate_input(self, searchtext, city=None, state_province=None, country=None): if searchtext and searchtext.strip(): @@ -88,8 +102,13 @@ class MapboxGeocoder(Traceable): :param country: Country ISO 3166 code :return: [x, y] on success, [] on error """ + return self.geocode_meta(searchtext, city, state_province, country)[0] + + @qps_retry(qps=10) + def geocode_meta(self, searchtext, city=None, state_province=None, + country=None): if not self._validate_input(searchtext, city, state_province, country): - return [] + return EMPTY_RESPONSE address = [] if searchtext and searchtext.strip(): @@ -99,32 +118,31 @@ class MapboxGeocoder(Traceable): if state_province: address.append(normalize(state_province)) - country = [country] if country else None - free_search = ', '.join(address) - return self.geocode_free_text([free_search], country)[0] + return self.geocode_free_text_meta([free_search], country)[0] @qps_retry(qps=10) - def geocode_free_text(self, free_searches, country=None): + def geocode_free_text_meta(self, free_searches, country=None): """ :param free_searches: Free text searches :param country: Country ISO 3166 code :return: list of [x, y] on success, [] on error """ + country = [country] if country else None + try: free_search = ';'.join([self._escape(fs) for fs in free_searches]) - self._logger.debug('--> free search: {}'.format(free_search)) + self._logger.debug('--> free search: {}, country: {}'.format(free_search, country)) response = self._geocoder.forward(address=free_search.decode('utf-8'), - country=country, - limit=1) + country=country) if response.status_code == requests.codes.ok: return self._parse_geocoder_response(response.text) elif response.status_code == requests.codes.bad_request: - return [] + return EMPTY_RESPONSE elif response.status_code == requests.codes.unprocessable_entity: - return [] + return EMPTY_RESPONSE else: raise ServiceException(response.status_code, response) except requests.Timeout as te: @@ -138,7 +156,7 @@ class MapboxGeocoder(Traceable): # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to Mapbox geocoding server', exception=ce) - return [] + return EMPTY_RESPONSE def _escape(self, free_search): # Semicolon is used to separate batch geocoding; there's no documented diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index e842396..b0c6deb 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -22,7 +22,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Madrid': [-3.7037902, 40.4167754], 'Logroño, Spain': [-2.4449852, 42.4627195], 'Logroño, Argentina': [-61.6961807, -29.5031057], - 'Plaza España 1, Barcelona': [2.1482563, 41.375485] + 'Plaza España, Barcelona': [2.1482563, 41.375485] } HERE_POINTS = { @@ -37,7 +37,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Madrid': [-3.70578, 40.42028], 'Logroño, Spain': [-2.45194, 42.46592], 'Logroño, Argentina': [-61.69604, -29.50425], - 'Plaza España 1, Barcelona': [2.1735699, 41.3823] # TODO: not ideal + 'Plaza España, Barcelona': [2.1735699, 41.3823] # TODO: not ideal } TOMTOM_POINTS = HERE_POINTS.copy() @@ -48,7 +48,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Valladolid, Spain': [-4.72838, 41.6542], 'Madrid': [-3.70035, 40.42028], 'Logroño, Spain': [-2.44998, 42.46592], - 'Plaza España 1, Barcelona': [2.07479, 41.36818] # TODO: not ideal + 'Plaza España, Barcelona': [2.07479, 41.36818] # TODO: not ideal }) MAPBOX_POINTS = GOOGLE_POINTS.copy() @@ -59,7 +59,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Valladolid, Spain': [-4.72856, 41.652251], '1902 amphitheatre parkway': [-118.03, 34.06], # TODO: huge mismatch 'Madrid': [-3.69194, 40.4167754], - 'Plaza España 1, Barcelona': [2.245969, 41.452483] # TODO: not ideal + 'Plaza España, Barcelona': [2.245969, 41.452483] # TODO: not ideal }) FIXTURE_POINTS = { @@ -69,6 +69,21 @@ class TestStreetFunctionsSetUp(TestCase): 'mapbox': MAPBOX_POINTS } + HERE_RELEVANCES = { + 'Plaza España, Barcelona': 1 + } + + MAPBOX_RELEVANCES = { + 'Plaza España, Barcelona': 0.75 + } + + RELEVANCES = { + 'here': HERE_RELEVANCES, + 'tomtom': HERE_RELEVANCES, + 'mapbox': MAPBOX_RELEVANCES, + 'google': HERE_RELEVANCES + } + def setUp(self): self.env_variables = IntegrationTestHelper.get_environment_variables() self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format( @@ -86,6 +101,8 @@ class TestStreetFunctionsSetUp(TestCase): provider = response['rows'][0]['provider'] self.fixture_points = self.FIXTURE_POINTS[provider] + self.relevances = self.RELEVANCES[provider] + def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, @@ -118,12 +135,12 @@ class TestStreetFunctions(TestStreetFunctionsSetUp): def test_component_aggregation(self): query = "select st_x(the_geom), st_y(the_geom) from (" \ "select cdb_dataservices_client.cdb_geocode_street_point( " \ - "'Plaza España 1', 'Barcelona', null, 'Spain') as the_geom) _x" + "'Plaza España', 'Barcelona', null, 'Spain') as the_geom) _x" response = self._run_authenticated(query) row = response['rows'][0] x_y = [row['st_x'], row['st_y']] # Wrong coordinates (Plaza España, Madrid): [-3.7138975, 40.4256762] - assert_close_enough(x_y, self.fixture_points['Plaza España 1, Barcelona']) + assert_close_enough(x_y, self.fixture_points['Plaza España, Barcelona']) class TestBulkStreetFunctions(TestStreetFunctionsSetUp): @@ -294,18 +311,18 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): x_y_by_cartodb_id = self._x_y_by_cartodb_id(response) assert_equal(x_y_by_cartodb_id[1], x_y_by_cartodb_id[2]) - # "'Plaza España 1', 'Barcelona', null, 'Spain') as the_geom) _x" + # "'Plaza España', 'Barcelona', null, 'Spain') as the_geom) _x" def test_component_aggregation(self): query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ "'select 1 as cartodb_id, ''Spain'' as country, " \ "''Barcelona'' as city, " \ - "''Plaza España 1'' as street' " \ + "''Plaza España'' as street' " \ ", 'street', 'city', NULL, 'country')" response = self._run_authenticated(query) assert_close_enough(self._x_y_by_cartodb_id(response)[1], - self.fixture_points['Plaza España 1, Barcelona']) + self.fixture_points['Plaza España, Barcelona']) def _test_known_table(self): subquery = 'select * from known_table where cartodb_id < 1100' @@ -325,11 +342,12 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ "'select 1 as cartodb_id, ''Spain'' as country, " \ "''Barcelona'' as city, " \ - "''Plaza España 1'' as street' " \ + "''Plaza España'' as street' " \ ", 'street', 'city', NULL, 'country')" response = self._run_authenticated(query) - assert_true(isclose(response['rows'][0]['metadata']['relevance'], 1)) + assert_true(isclose(response['rows'][0]['metadata']['relevance'], + self.relevances['Plaza España, Barcelona'])) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From 0b2ee85c1167793cfbc4216e71f3da2e2b77a1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 09:30:28 +0200 Subject: [PATCH 114/265] TomTom normalization --- .../cartodb_services/tomtom/geocoder.py | 7 +++++- test/integration/test_street_functions.py | 22 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index b064ea5..4b3843b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -4,6 +4,7 @@ import json import requests from uritemplate import URITemplate +from math import tanh from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -20,6 +21,7 @@ ENTRY_LON = 'lon' ENTRY_LAT = 'lat' EMPTY_RESPONSE = [[], {}] +SCORE_NORMALIZATION_FACTOR = 0.15 class TomTomGeocoder(Traceable): ''' @@ -132,5 +134,8 @@ class TomTomGeocoder(Traceable): def _extract_metadata_from_result(self, result): return { - 'relevance': result['score'] # TODO: normalize + 'relevance': self._normalize_score(result['score']) } + + def _normalize_score(self, score): + return tanh(score * SCORE_NORMALIZATION_FACTOR) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index b0c6deb..79af788 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -73,15 +73,21 @@ class TestStreetFunctionsSetUp(TestCase): 'Plaza España, Barcelona': 1 } - MAPBOX_RELEVANCES = { + MAPBOX_RELEVANCES = HERE_RELEVANCES.copy() + MAPBOX_RELEVANCES.update({ 'Plaza España, Barcelona': 0.75 - } + }) + + TOMTOM_RELEVANCES = MAPBOX_RELEVANCES.copy() + TOMTOM_RELEVANCES.update({ + 'Plaza España, Barcelona': 0.85 + }) RELEVANCES = { + 'google': HERE_RELEVANCES, 'here': HERE_RELEVANCES, - 'tomtom': HERE_RELEVANCES, - 'mapbox': MAPBOX_RELEVANCES, - 'google': HERE_RELEVANCES + 'tomtom': TOMTOM_RELEVANCES, + 'mapbox': MAPBOX_RELEVANCES } def setUp(self): @@ -346,8 +352,10 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): ", 'street', 'city', NULL, 'country')" response = self._run_authenticated(query) - assert_true(isclose(response['rows'][0]['metadata']['relevance'], - self.relevances['Plaza España, Barcelona'])) + relevance = response['rows'][0]['metadata']['relevance'] + expected_relevance = self.relevances['Plaza España, Barcelona'] + assert_true(isclose(relevance, expected_relevance, 0.05), + '{} not close to {}'.format(relevance, expected_relevance)) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, From d46d51c3bbf49608b038499015f4ae66ee1ca7c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 11:43:54 +0200 Subject: [PATCH 115/265] Relevance metadata for Google --- .../cartodb_services/google/bulk_geocoder.py | 29 ++++-------- .../cartodb_services/google/geocoder.py | 46 ++++++++++++++++--- test/integration/test_street_functions.py | 9 +++- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index e59e408..42a60d9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -6,8 +6,7 @@ from cartodb_services.google import GoogleMapsGeocoder def async_geocoder(geocoder, address, components): - results = geocoder.geocode(address=address, components=components) - return results if results else [] + return geocoder.geocode(address=address, components=components) class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): @@ -26,12 +25,9 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (cartodb_id, street, city, state, country) = search - address = compose_address(street, city, state, country) - components = self._build_optional_parameters(city, state, country) - result = self.geocoder.geocode(address=address, components=components) - lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] - self._logger.debug('--> lng_lat: {}'.format(lng_lat)) - results.append((cartodb_id, lng_lat, [])) + lng_lat, metadata = self.geocode_meta(street, city, state, country) + self._logger.debug('--> lng_lat: {}. metadata: {}'.format(lng_lat, metadata)) + results.append((cartodb_id, lng_lat, metadata)) return results def _batch_geocode(self, searches): @@ -39,16 +35,13 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): pool = Pool(processes=self.PARALLEL_PROCESSES) for search in searches: (cartodb_id, street, city, state, country) = search - components = self._build_optional_parameters(city, state, country) - # Geocoding works better if components are also inside the address + self._logger.debug('async geocoding --> {}'.format(search)) address = compose_address(street, city, state, country) if address: - self._logger.debug('async geocoding --> {} {}'.format(address.encode('utf-8'), components)) + components = self._build_optional_parameters(city, state, country) result = pool.apply_async(async_geocoder, (self.geocoder, address, components)) - else: - result = [] - bulk_results[cartodb_id] = result + bulk_results[cartodb_id] = result pool.close() pool.join() @@ -56,13 +49,12 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): results = [] for cartodb_id, bulk_result in bulk_results.items(): try: - result = bulk_result.get() + lng_lat, metadata = self._process_results(bulk_result.get()) except Exception as e: self._logger.error('Error at Google async_geocoder', e) - result = [] + lng_lat, metadata = [[], {}] - lng_lat = self._extract_lng_lat_from_result(result[0]) if result else [] - results.append((cartodb_id, lng_lat, [])) + results.append((cartodb_id, lng_lat, metadata)) return results except KeyError as e: self._logger.error('KeyError error', exception=e) @@ -70,4 +62,3 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): except Exception as e: self._logger.error('General error', exception=e) raise e - diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index aa8877c..b07dc6b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -8,6 +8,15 @@ from cartodb_services.geocoder import compose_address from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory +EMPTY_RESPONSE = [[], {}] +PARTIAL_FACTOR = 0.8 +RELEVANCE_BY_LOCATION_TYPE = { + 'ROOFTOP': 1, + 'GEOMETRIC_CENTER': 0.9, + 'RANGE_INTERPOLATED': 0.8, + 'APPROXIMATE': 0.7 +} + class GoogleMapsGeocoder(): @@ -19,26 +28,49 @@ class GoogleMapsGeocoder(): self.geocoder = GoogleMapsClientFactory.get(self.client_id, self.client_secret, self.channel) self._logger = logger - def geocode(self, searchtext, city=None, state=None, - country=None): + def geocode(self, searchtext, city=None, state=None, country=None): + return self.geocode_meta(searchtext, city, state, country)[0] + + def geocode_meta(self, searchtext, city=None, state=None, country=None): try: address = compose_address(searchtext, city, state, country) opt_params = self._build_optional_parameters(city, state, country) results = self.geocoder.geocode(address=address, components=opt_params) - if results: - return self._extract_lng_lat_from_result(results[0]) - else: - return [] - except KeyError: + return self._process_results(results) + except KeyError as e: + self._logger.error('params: {}, {}, {}, {}'.format( + searchtext.encode('utf-8'), city.encode('utf-8'), + state.encode('utf-8'), country.encode('utf-8') + ), e) raise MalformedResult() + def _process_results(self, results): + if results: + self._logger.debug('--> results: {}'.format(results[0])) + return [ + self._extract_lng_lat_from_result(results[0]), + self._extract_metadata_from_result(results[0]) + ] + else: + return EMPTY_RESPONSE + def _extract_lng_lat_from_result(self, result): location = result['geometry']['location'] longitude = location['lng'] latitude = location['lat'] return [longitude, latitude] + def _extract_metadata_from_result(self, result): + location_type = result['geometry']['location_type'] + base_relevance = RELEVANCE_BY_LOCATION_TYPE[location_type] + partial_match = result.get('partial_match', False) + partial_factor = PARTIAL_FACTOR if partial_match else 1 + return { + 'relevance': base_relevance * partial_factor + } + + def _build_optional_parameters(self, city=None, state=None, country=None): optional_params = {} diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 79af788..cca42ca 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -83,8 +83,13 @@ class TestStreetFunctionsSetUp(TestCase): 'Plaza España, Barcelona': 0.85 }) + GOOGLE_RELEVANCES = HERE_RELEVANCES.copy() + GOOGLE_RELEVANCES.update({ + 'Plaza España, Barcelona': 0.9 + }) + RELEVANCES = { - 'google': HERE_RELEVANCES, + 'google': GOOGLE_RELEVANCES, 'here': HERE_RELEVANCES, 'tomtom': TOMTOM_RELEVANCES, 'mapbox': MAPBOX_RELEVANCES @@ -331,7 +336,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self.fixture_points['Plaza España, Barcelona']) def _test_known_table(self): - subquery = 'select * from known_table where cartodb_id < 1100' + subquery = 'select * from unknown_table where cartodb_id < 1100' subquery_count = 'select count(1) from ({}) _x'.format(subquery) count = self._run_authenticated(subquery_count)['rows'][0]['count'] From da78b0bc654986679a9599577a2740b6822cbdc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 12:28:39 +0200 Subject: [PATCH 116/265] Fix batching with negatives cartodb_id --- client/cdb_dataservices_client--0.25.0.sql | 13 +++++++------ client/sql/21_bulk_geocoding_functions.sql | 13 +++++++------ test/integration/test_street_functions.py | 9 +++++++-- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 2bed351..af58ce2 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1995,6 +1995,8 @@ DECLARE enough_quota boolean; remaining_quota integer; max_batch_size integer; + max_cartodb_id integer; + min_cartodb_id integer; cartodb_id_batch integer; batches_n integer; @@ -2020,7 +2022,8 @@ BEGIN batch_size := MAX_SAFE_BATCH_SIZE; END IF; - EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; + EXECUTE format('SELECT count(1), max(cartodb_id), min(cartodb_id), ceil((max(cartodb_id) + 1 - min(cartodb_id))::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, max_cartodb_id, min_cartodb_id, batches_n; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; @@ -2029,8 +2032,6 @@ BEGIN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; - EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', batch_size, query) INTO batches_n; - RAISE DEBUG 'batches_n: %', batches_n; temp_table_name := 'bulk_geocode_street_' || md5(random()::text); @@ -2052,13 +2053,13 @@ BEGIN 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id-1)::float/$1) as batch' || + ' floor((cartodb_id - $1)::float/$2) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING batch_size, cartodb_id_batch; + 'WHERE batch = $3', street_column, city_column, state_column, country_column, query, temp_table_name) + USING min_cartodb_id, batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index f9ba824..7ab3407 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -6,6 +6,8 @@ DECLARE enough_quota boolean; remaining_quota integer; max_batch_size integer; + max_cartodb_id integer; + min_cartodb_id integer; cartodb_id_batch integer; batches_n integer; @@ -31,7 +33,8 @@ BEGIN batch_size := MAX_SAFE_BATCH_SIZE; END IF; - EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; + EXECUTE format('SELECT count(1), max(cartodb_id), min(cartodb_id), ceil((max(cartodb_id) + 1 - min(cartodb_id))::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, max_cartodb_id, min_cartodb_id, batches_n; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; @@ -40,8 +43,6 @@ BEGIN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; - EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', batch_size, query) INTO batches_n; - RAISE DEBUG 'batches_n: %', batches_n; temp_table_name := 'bulk_geocode_street_' || md5(random()::text); @@ -63,13 +64,13 @@ BEGIN 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id-1)::float/$1) as batch' || + ' floor((cartodb_id - $1)::float/$2) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) - USING batch_size, cartodb_id_batch; + 'WHERE batch = $3', street_column, city_column, state_column, country_column, query, temp_table_name) + USING min_cartodb_id, batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index cca42ca..43186b9 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -287,11 +287,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): Useful just to test a good batch size """ n = 110 + first_cartodb_id = -1 + first_street_number = 1 batch_size = 'NULL' # NULL for optimal streets = [] for i in range(0, n): streets.append('{{"cartodb_id": {}, "address": "{} Yonge Street, ' \ - 'Toronto, Canada"}}'.format(i, i)) + 'Toronto, Canada"}}'.format(first_cartodb_id + i, + first_street_number + i)) query = "select *, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ @@ -300,7 +303,9 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "]''::jsonb) as (cartodb_id integer, address text)', " \ "'address', null, null, null, {})".format(','.join(streets), batch_size) response = self._run_authenticated(query) - assert_equal(n - 1, len(response['rows'])) + assert_equal(n, len(response['rows'])) + for row in response['rows']: + assert_not_equal(row['st_x'], None) def test_missing_components_on_private_function(self): query = "SELECT _cdb_bulk_geocode_street_point(" \ From b77974258564f2ff2b053438a442283dc24a5a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 12:51:56 +0200 Subject: [PATCH 117/265] Fix logging on error --- .../cartodb_services/cartodb_services/google/geocoder.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index b07dc6b..491ce0f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -32,17 +32,14 @@ class GoogleMapsGeocoder(): return self.geocode_meta(searchtext, city, state, country)[0] def geocode_meta(self, searchtext, city=None, state=None, country=None): + address = compose_address(searchtext, city, state, country) try: - address = compose_address(searchtext, city, state, country) opt_params = self._build_optional_parameters(city, state, country) results = self.geocoder.geocode(address=address, components=opt_params) return self._process_results(results) except KeyError as e: - self._logger.error('params: {}, {}, {}, {}'.format( - searchtext.encode('utf-8'), city.encode('utf-8'), - state.encode('utf-8'), country.encode('utf-8') - ), e) + self._logger.error('address: {}'.format(address), e) raise MalformedResult() def _process_results(self, results): From 67fee1cce897bafe6a60ea5f6a18790055208212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 13:59:48 +0200 Subject: [PATCH 118/265] Precision metadata for Google --- .../cartodb_services/__init__.py | 3 + .../cartodb_services/google/geocoder.py | 10 ++- test/integration/test_street_functions.py | 75 ++++++++++++------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index ef5caa4..dd364b4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -35,3 +35,6 @@ def _reset(): GD = None from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder + +PRECISION_PRECISE = 'precise' +PRECISION_INTERPOLATED = 'interpolated' diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 491ce0f..f4cd0a5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -4,6 +4,7 @@ from urlparse import parse_qs from exceptions import MalformedResult +from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.geocoder import compose_address from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory @@ -16,6 +17,12 @@ RELEVANCE_BY_LOCATION_TYPE = { 'RANGE_INTERPOLATED': 0.8, 'APPROXIMATE': 0.7 } +PRECISION_BY_LOCATION_TYPE = { + 'ROOFTOP': PRECISION_PRECISE, + 'GEOMETRIC_CENTER': PRECISION_PRECISE, + 'RANGE_INTERPOLATED': PRECISION_INTERPOLATED, + 'APPROXIMATE': PRECISION_INTERPOLATED +} class GoogleMapsGeocoder(): @@ -64,7 +71,8 @@ class GoogleMapsGeocoder(): partial_match = result.get('partial_match', False) partial_factor = PARTIAL_FACTOR if partial_match else 1 return { - 'relevance': base_relevance * partial_factor + 'relevance': base_relevance * partial_factor, + 'precision': PRECISION_BY_LOCATION_TYPE[location_type] } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 43186b9..1b8430d 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -69,30 +69,38 @@ class TestStreetFunctionsSetUp(TestCase): 'mapbox': MAPBOX_POINTS } - HERE_RELEVANCES = { - 'Plaza España, Barcelona': 1 + GOOGLE_METADATAS = { + 'Plaza España, Barcelona': { + 'relevance': 0.9, 'precision': 'precise' + }, + 'Santiago Rusiñol 123, Valladolid': { + 'relevance': 0.8, 'precision': 'interpolated' + } } - MAPBOX_RELEVANCES = HERE_RELEVANCES.copy() - MAPBOX_RELEVANCES.update({ - 'Plaza España, Barcelona': 0.75 - }) + HERE_METADATAS = { + 'Plaza España, Barcelona': { + 'relevance': 1 + } + } - TOMTOM_RELEVANCES = MAPBOX_RELEVANCES.copy() - TOMTOM_RELEVANCES.update({ - 'Plaza España, Barcelona': 0.85 - }) + TOMTOM_METADATAS = { + 'Plaza España, Barcelona': { + 'relevance': 0.85 + } + } - GOOGLE_RELEVANCES = HERE_RELEVANCES.copy() - GOOGLE_RELEVANCES.update({ - 'Plaza España, Barcelona': 0.9 - }) + MAPBOX_METADATAS = { + 'Plaza España, Barcelona': { + 'relevance': 0.75 + } + } - RELEVANCES = { - 'google': GOOGLE_RELEVANCES, - 'here': HERE_RELEVANCES, - 'tomtom': TOMTOM_RELEVANCES, - 'mapbox': MAPBOX_RELEVANCES + METADATAS = { + 'google': GOOGLE_METADATAS, + 'here': HERE_METADATAS, + 'tomtom': TOMTOM_METADATAS, + 'mapbox': MAPBOX_METADATAS } def setUp(self): @@ -112,7 +120,7 @@ class TestStreetFunctionsSetUp(TestCase): provider = response['rows'][0]['provider'] self.fixture_points = self.FIXTURE_POINTS[provider] - self.relevances = self.RELEVANCES[provider] + self.metadata = self.METADATAS[provider] def _run_authenticated(self, query): @@ -353,19 +361,25 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(len(response['rows']), count) assert_not_equal(response['rows'][0]['st_x'], None) - def test_relevance(self): + def test_metadata(self): query = "select metadata " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ "'select 1 as cartodb_id, ''Spain'' as country, " \ "''Barcelona'' as city, " \ - "''Plaza España'' as street' " \ + "''Plaza España'' as street " \ + "UNION " \ + "select 2 as cartodb_id, ''Spain'' as country, " \ + "''Valladolid'' as city, " \ + "''Calle Santiago Rusiñol 123'' as street' " \ ", 'street', 'city', NULL, 'country')" response = self._run_authenticated(query) - relevance = response['rows'][0]['metadata']['relevance'] - expected_relevance = self.relevances['Plaza España, Barcelona'] - assert_true(isclose(relevance, expected_relevance, 0.05), - '{} not close to {}'.format(relevance, expected_relevance)) + expected = [ + self.metadata['Plaza España, Barcelona'], + self.metadata['Santiago Rusiñol 123, Valladolid'] + ] + for r, e in zip(response['rows'], expected): + self.assert_metadata(r['metadata'], e) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, @@ -384,3 +398,12 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(len(points_a_by_cartodb_id), len(points_b_by_cartodb_id)) for cartodb_id, point in points_a_by_cartodb_id.iteritems(): assert_close_enough(point, points_b_by_cartodb_id[cartodb_id]) + + @staticmethod + def assert_metadata(metadata, expected): + relevance = metadata['relevance'] + expected_relevance = expected['relevance'] + assert_true(isclose(relevance, expected_relevance, 0.05), + '{} not close to {}'.format(relevance, expected_relevance)) + + assert_equal(metadata['precision'], expected['precision']) From dbb4f9204aabd2ea8a9cd78ab753f12607006d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 15:30:51 +0200 Subject: [PATCH 119/265] Precision metadata for HERE --- .../cartodb_services/here/geocoder.py | 16 ++++++++++++++-- test/integration/test_street_functions.py | 19 +++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 8139e1a..44d695d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,8 +6,14 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * +from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable +PRECISION_BY_MATCH_TYPE = { + 'pointAddress': PRECISION_PRECISE, + 'interpolated': PRECISION_INTERPOLATED +} + class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' @@ -82,11 +88,13 @@ class HereMapsGeocoder(Traceable): try: response = self._perform_request(params) result = response['Response']['View'][0]['Result'][0] + self._logger.debug('--> Result: {}'.format(result)) return [self._extract_lng_lat_from_result(result), self._extract_metadata_from_result(result)] except IndexError: return [[], {}] - except KeyError: + except KeyError as e: + self._logger.error('params: {}'.format(params), e) raise MalformedResult() def _perform_request(self, params): @@ -124,6 +132,10 @@ class HereMapsGeocoder(Traceable): return [longitude, latitude] def _extract_metadata_from_result(self, result): + # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses + precision = PRECISION_BY_MATCH_TYPE[ + result.get('MatchType', 'pointAddress')] return { - 'relevance': result['Relevance'] + 'relevance': result['Relevance'], + 'precision': precision } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 1b8430d..cfc4547 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -70,18 +70,17 @@ class TestStreetFunctionsSetUp(TestCase): } GOOGLE_METADATAS = { - 'Plaza España, Barcelona': { - 'relevance': 0.9, 'precision': 'precise' - }, - 'Santiago Rusiñol 123, Valladolid': { - 'relevance': 0.8, 'precision': 'interpolated' - } + 'Plaza España, Barcelona': + {'relevance': 0.9, 'precision': 'precise'}, + 'Santiago Rusiñol 123, Valladolid': + {'relevance': 0.8, 'precision': 'interpolated'} } HERE_METADATAS = { - 'Plaza España, Barcelona': { - 'relevance': 1 - } + 'Plaza España, Barcelona': + {'relevance': 1, 'precision': 'precise'}, + 'Santiago Rusiñol 123, Valladolid': + {'relevance': 0.89, 'precision': 'precise'} # Wrong. See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses } TOMTOM_METADATAS = { @@ -98,7 +97,7 @@ class TestStreetFunctionsSetUp(TestCase): METADATAS = { 'google': GOOGLE_METADATAS, - 'here': HERE_METADATAS, + 'heremaps': HERE_METADATAS, 'tomtom': TOMTOM_METADATAS, 'mapbox': MAPBOX_METADATAS } From 4123a4c4425e96996c1c30da7446aaefaff9e850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 19:09:02 +0200 Subject: [PATCH 120/265] Precision metadata for TomTom --- .../cartodb_services/tomtom/geocoder.py | 10 +++++++++- test/integration/test_street_functions.py | 11 ++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 4b3843b..42970f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -5,6 +5,7 @@ import json import requests from uritemplate import URITemplate from math import tanh +from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -22,6 +23,7 @@ ENTRY_LAT = 'lat' EMPTY_RESPONSE = [[], {}] SCORE_NORMALIZATION_FACTOR = 0.15 +PRECISION_SCORE_THRESHOLD = 0.5 class TomTomGeocoder(Traceable): ''' @@ -133,9 +135,15 @@ class TomTomGeocoder(Traceable): return EMPTY_RESPONSE def _extract_metadata_from_result(self, result): + score = self._normalize_score(result['score']) return { - 'relevance': self._normalize_score(result['score']) + 'relevance': score, + 'precision': self._precision_from_score(score) } def _normalize_score(self, score): return tanh(score * SCORE_NORMALIZATION_FACTOR) + + def _precision_from_score(self, score): + return PRECISION_PRECISE \ + if score > PRECISION_SCORE_THRESHOLD else PRECISION_INTERPOLATED diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index cfc4547..970e1f7 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -73,7 +73,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Plaza España, Barcelona': {'relevance': 0.9, 'precision': 'precise'}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.8, 'precision': 'interpolated'} + {'relevance': 0.56, 'precision': 'interpolated'} } HERE_METADATAS = { @@ -84,9 +84,10 @@ class TestStreetFunctionsSetUp(TestCase): } TOMTOM_METADATAS = { - 'Plaza España, Barcelona': { - 'relevance': 0.85 - } + 'Plaza España, Barcelona': + {'relevance': 0.85, 'precision': 'precise'}, + 'Santiago Rusiñol 123, Valladolid': + {'relevance': 0.448283494533, 'precision': 'interpolated'} } MAPBOX_METADATAS = { @@ -369,7 +370,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "UNION " \ "select 2 as cartodb_id, ''Spain'' as country, " \ "''Valladolid'' as city, " \ - "''Calle Santiago Rusiñol 123'' as street' " \ + "''Santiago Rusiñol 123'' as street' " \ ", 'street', 'city', NULL, 'country')" response = self._run_authenticated(query) From 6e78da55b27d2f814e23a6645cb6b956998f5949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 11 Jul 2018 19:28:16 +0200 Subject: [PATCH 121/265] Precision metadata for Mapbox --- .../cartodb_services/cartodb_services/mapbox/geocoder.py | 9 ++++++++- test/integration/test_street_functions.py | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 5d644d9..9eb5e4e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -5,6 +5,7 @@ Python client for the Mapbox Geocoder service. import json import requests from mapbox import Geocoder +from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -74,8 +75,14 @@ class MapboxGeocoder(Traceable): return [longitude, latitude] def _extract_metadata_from_result(self, result): + if result[ENTRY_GEOMETRY].get('interpolated', False): + precision = PRECISION_INTERPOLATED + else: + precision = PRECISION_PRECISE + return { - 'relevance': self._normalize_relevance(float(result['relevance'])) + 'relevance': self._normalize_relevance(float(result['relevance'])), + 'precision': precision } def _normalize_relevance(self, relevance): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 970e1f7..c8abeae 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -91,9 +91,10 @@ class TestStreetFunctionsSetUp(TestCase): } MAPBOX_METADATAS = { - 'Plaza España, Barcelona': { - 'relevance': 0.75 - } + 'Plaza España, Barcelona': + {'relevance': 0.666, 'precision': 'precise'}, + 'Santiago Rusiñol 123, Valladolid': + {'relevance': 0.666, 'precision': 'precise'} # TODO: wrong } METADATAS = { From f2197d4b2ae3e35b4ac12ec1fea31fd1082f4fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 11:34:33 +0200 Subject: [PATCH 122/265] match_types for Google metadata --- .../cartodb_services/google/geocoder.py | 17 ++++++++++++++++- test/integration/test_street_functions.py | 6 ++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index f4cd0a5..74d8f53 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -23,6 +23,18 @@ PRECISION_BY_LOCATION_TYPE = { 'RANGE_INTERPOLATED': PRECISION_INTERPOLATED, 'APPROXIMATE': PRECISION_INTERPOLATED } +MATCH_TYPE_BY_MATCH_LEVEL = { + 'point_of_interest': 'point_of_interest', + 'country': 'country', + 'administrative_area_level_1': 'state', + 'administrative_area_level_2': 'county', + 'locality': 'locality', + 'sublocality': 'district', + 'street_address': 'street', + 'intersection': 'intersection', + 'street_number': 'street_number', + 'postal_code': 'postal_code' +} class GoogleMapsGeocoder(): @@ -70,9 +82,12 @@ class GoogleMapsGeocoder(): base_relevance = RELEVANCE_BY_LOCATION_TYPE[location_type] partial_match = result.get('partial_match', False) partial_factor = PARTIAL_FACTOR if partial_match else 1 + match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) + for match_level in result['types']] return { 'relevance': base_relevance * partial_factor, - 'precision': PRECISION_BY_LOCATION_TYPE[location_type] + 'precision': PRECISION_BY_LOCATION_TYPE[location_type], + 'match_types': filter(None, match_types) } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index c8abeae..7575b7c 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -71,9 +71,9 @@ class TestStreetFunctionsSetUp(TestCase): GOOGLE_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.9, 'precision': 'precise'}, + {'relevance': 0.9, 'precision': 'precise', 'match_types': ['point_of_interest']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.56, 'precision': 'interpolated'} + {'relevance': 0.56, 'precision': 'interpolated', 'match_types': ['locality']} } HERE_METADATAS = { @@ -408,3 +408,5 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): '{} not close to {}'.format(relevance, expected_relevance)) assert_equal(metadata['precision'], expected['precision']) + + assert_equal(metadata['match_types'], expected['match_types']) From 0b635377ef542b86d40c52a081829b224b4f38e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 11:52:25 +0200 Subject: [PATCH 123/265] match_types for HERE metadata --- .../cartodb_services/here/geocoder.py | 16 +++++++++++++++- test/integration/test_street_functions.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 44d695d..d8f7f41 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -13,6 +13,18 @@ PRECISION_BY_MATCH_TYPE = { 'pointAddress': PRECISION_PRECISE, 'interpolated': PRECISION_INTERPOLATED } +MATCH_TYPE_BY_MATCH_LEVEL = { + 'landmark': 'point_of_interest', + 'country': 'country', + 'state': 'state', + 'county': 'county', + 'city': 'locality', + 'district': 'district', + 'street': 'street', + 'intersection': 'intersection', + 'houseNumber': 'street_number', + 'postalCode': 'postal_code' +} class HereMapsGeocoder(Traceable): @@ -135,7 +147,9 @@ class HereMapsGeocoder(Traceable): # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses precision = PRECISION_BY_MATCH_TYPE[ result.get('MatchType', 'pointAddress')] + match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) return { 'relevance': result['Relevance'], - 'precision': precision + 'precision': precision, + 'match_types': [match_type] if match_type else [] } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 7575b7c..bf7672e 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -78,9 +78,9 @@ class TestStreetFunctionsSetUp(TestCase): HERE_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 1, 'precision': 'precise'}, + {'relevance': 1, 'precision': 'precise', 'match_types': ['street']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.89, 'precision': 'precise'} # Wrong. See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses + {'relevance': 0.89, 'precision': 'precise', 'match_types': ['street']} # Wrong. See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses } TOMTOM_METADATAS = { From 0a92ae14452331fc0a3fee960e06a2daecc8aee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 12:01:55 +0200 Subject: [PATCH 124/265] match_types for TomTom metadata --- .../cartodb_services/tomtom/geocoder.py | 11 ++++++++++- test/integration/test_street_functions.py | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 42970f8..09ed69d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -24,6 +24,13 @@ EMPTY_RESPONSE = [[], {}] SCORE_NORMALIZATION_FACTOR = 0.15 PRECISION_SCORE_THRESHOLD = 0.5 +MATCH_TYPE_BY_MATCH_LEVEL = { + 'POI': 'point_of_interest', + 'Street': 'street', + 'Address Range': 'street', + 'Cross Street': 'intersection', + 'Point Address': 'street_number' +} class TomTomGeocoder(Traceable): ''' @@ -136,9 +143,11 @@ class TomTomGeocoder(Traceable): def _extract_metadata_from_result(self, result): score = self._normalize_score(result['score']) + match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['type'], None) return { 'relevance': score, - 'precision': self._precision_from_score(score) + 'precision': self._precision_from_score(score), + 'match_types': [match_type] if match_type else [] } def _normalize_score(self, score): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index bf7672e..1c05058 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase): TOMTOM_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.85, 'precision': 'precise'}, + {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.448283494533, 'precision': 'interpolated'} + {'relevance': 0.448283494533, 'precision': 'interpolated', 'match_types': ['street']} } MAPBOX_METADATAS = { From 080de34163c3fd9b3793c357131bec8f19dedb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 12:11:40 +0200 Subject: [PATCH 125/265] match_types for Mapbox metadata --- .../cartodb_services/mapbox/geocoder.py | 16 +++++++++++++++- test/integration/test_street_functions.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 9eb5e4e..1fc8b4e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -25,6 +25,17 @@ TYPE_POINT = 'Point' EMPTY_RESPONSE = [[], {}] +MATCH_TYPE_BY_MATCH_LEVEL = { + 'poi': 'point_of_interest', + 'poi.landmark': 'point_of_interest', + 'place': 'point_of_interest', + 'country': 'country', + 'region': 'state', + 'locality': 'locality', + 'district': 'district', + 'address': 'street' +} + class MapboxGeocoder(Traceable): ''' @@ -80,9 +91,12 @@ class MapboxGeocoder(Traceable): else: precision = PRECISION_PRECISE + match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) + for match_level in result['place_type']] return { 'relevance': self._normalize_relevance(float(result['relevance'])), - 'precision': precision + 'precision': precision, + 'match_types': filter(None, match_types) } def _normalize_relevance(self, relevance): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 1c05058..86d03c6 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -92,9 +92,9 @@ class TestStreetFunctionsSetUp(TestCase): MAPBOX_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.666, 'precision': 'precise'}, + {'relevance': 0.666, 'precision': 'precise', 'match_types': ['point_of_interest']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.666, 'precision': 'precise'} # TODO: wrong + {'relevance': 0.666, 'precision': 'precise', 'match_types': ['point_of_interest']} # TODO: wrong } METADATAS = { From e82346e7f68c286225b19f828fc580e38429140b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 12:43:40 +0200 Subject: [PATCH 126/265] match_types for batched HERE metadata --- .../cartodb_services/here/bulk_geocoder.py | 7 +++- .../cartodb_services/here/geocoder.py | 39 +++++++++---------- test/integration/test_street_functions.py | 5 +++ 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index dd47fda..68c4766 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -140,10 +140,15 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): reader = csv.DictReader(root_zip.open(name), delimiter='|') for row in reader: if row['SeqNumber'] == '1': # First per requested data + precision = self.PRECISION_BY_MATCH_TYPE[ + row.get('matchType', 'pointAddress')] + match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None) results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']], { - 'relevance': float(row['relevance']) + 'relevance': float(row['relevance']), + 'precision': precision, + 'match_types': [match_type] if match_type else [] })) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index d8f7f41..6031c97 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -9,24 +9,6 @@ from exceptions import * from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable -PRECISION_BY_MATCH_TYPE = { - 'pointAddress': PRECISION_PRECISE, - 'interpolated': PRECISION_INTERPOLATED -} -MATCH_TYPE_BY_MATCH_LEVEL = { - 'landmark': 'point_of_interest', - 'country': 'country', - 'state': 'state', - 'county': 'county', - 'city': 'locality', - 'district': 'district', - 'street': 'street', - 'intersection': 'intersection', - 'houseNumber': 'street_number', - 'postalCode': 'postal_code' -} - - class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' @@ -70,6 +52,23 @@ class HereMapsGeocoder(Traceable): 'strictlanguagemode' ] + ADDRESS_PARAMS + PRECISION_BY_MATCH_TYPE = { + 'pointAddress': PRECISION_PRECISE, + 'interpolated': PRECISION_INTERPOLATED + } + MATCH_TYPE_BY_MATCH_LEVEL = { + 'landmark': 'point_of_interest', + 'country': 'country', + 'state': 'state', + 'county': 'county', + 'city': 'locality', + 'district': 'district', + 'street': 'street', + 'intersection': 'intersection', + 'houseNumber': 'street_number', + 'postalCode': 'postal_code' + } + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS): service_params = service_params or {} self.app_id = app_id @@ -145,9 +144,9 @@ class HereMapsGeocoder(Traceable): def _extract_metadata_from_result(self, result): # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses - precision = PRECISION_BY_MATCH_TYPE[ + precision = self.PRECISION_BY_MATCH_TYPE[ result.get('MatchType', 'pointAddress')] - match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) + match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) return { 'relevance': result['Relevance'], 'precision': precision, diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 86d03c6..9b134e7 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -315,6 +315,11 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(n, len(response['rows'])) for row in response['rows']: assert_not_equal(row['st_x'], None) + assert_not_equal(row['metadata'], {}) + metadata = row['metadata'] + assert_not_equal(metadata['relevance'], None) + assert_not_equal(metadata['precision'], None) + assert_not_equal(metadata['match_types'], None) def test_missing_components_on_private_function(self): query = "SELECT _cdb_bulk_geocode_street_point(" \ From 8cb9e123b1a625b8bd3e84d8ec9c322a6ee62949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 16 Jul 2018 19:55:04 +0200 Subject: [PATCH 127/265] Helper function to convert json arrays to PG arrays --- client/cdb_dataservices_client--0.25.0.sql | 7 ++++++- client/sql/05_utils.sql | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index af58ce2..b69a536 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -18,7 +18,12 @@ BEGIN RETURN ROWS; END -$func$ LANGUAGE plpgsql;-- +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE;-- -- Geocoder server connection config -- -- The purpose of this function is provide to the PL/Proxy functions diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index d161c6a..31c8445 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -12,4 +12,9 @@ BEGIN RETURN ROWS; END -$func$ LANGUAGE plpgsql; \ No newline at end of file +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE; \ No newline at end of file From e2762a6e0352a4bde5187603110498cc8b78054e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 10:06:43 +0200 Subject: [PATCH 128/265] Removed debug traces --- .../python/cartodb_services/cartodb_services/geocoder.py | 2 -- .../cartodb_services/google/bulk_geocoder.py | 2 -- .../cartodb_services/cartodb_services/google/geocoder.py | 1 - .../cartodb_services/here/bulk_geocoder.py | 8 -------- .../cartodb_services/cartodb_services/here/geocoder.py | 1 - .../cartodb_services/mapbox/bulk_geocoder.py | 4 ---- .../cartodb_services/cartodb_services/mapbox/geocoder.py | 2 -- .../cartodb_services/tomtom/bulk_geocoder.py | 8 -------- 8 files changed, 28 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index cea8ef4..c6043e4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -91,10 +91,8 @@ class StreetPointBulkGeocoder: if len(street_geocoder_searches) > self.MAX_BATCH_SIZE: raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) if self._should_use_batch(street_geocoder_searches): - self._logger.debug('--> Batch geocode') return self._batch_geocode(street_geocoder_searches) else: - self._logger.debug('--> Serial geocode') return self._serial_geocode(street_geocoder_searches) def _batch_geocode(self, street_geocoder_searches): diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index 42a60d9..0e1ab1f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -26,7 +26,6 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): for search in searches: (cartodb_id, street, city, state, country) = search lng_lat, metadata = self.geocode_meta(street, city, state, country) - self._logger.debug('--> lng_lat: {}. metadata: {}'.format(lng_lat, metadata)) results.append((cartodb_id, lng_lat, metadata)) return results @@ -35,7 +34,6 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): pool = Pool(processes=self.PARALLEL_PROCESSES) for search in searches: (cartodb_id, street, city, state, country) = search - self._logger.debug('async geocoding --> {}'.format(search)) address = compose_address(street, city, state, country) if address: components = self._build_optional_parameters(city, state, country) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 74d8f53..44b678c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -63,7 +63,6 @@ class GoogleMapsGeocoder(): def _process_results(self, results): if results: - self._logger.debug('--> results: {}'.format(results[0])) return [ self._extract_lng_lat_from_result(results[0]), self._extract_metadata_from_result(results[0]) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 68c4766..51b9ec9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -47,7 +47,6 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): def _batch_geocode(self, searches): request_id = self._send_batch(self._searches_to_csv(searches)) - self._logger.debug('--> Sent batch {}'.format(request_id)) last_processed = 0 stalled_retries = 0 @@ -55,26 +54,19 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): while True: job_info = self._job_status(request_id) if job_info.processed_count == last_processed: - self._logger.debug('--> no progress ({})'.format(last_processed)) stalled_retries += 1 if stalled_retries > self.MAX_STALLED_RETRIES: raise Exception('Too many retries for job {}'.format(request_id)) else: - self._logger.debug('--> progress ({} != {})'.format(job_info.processed_count, last_processed)) stalled_retries = 0 last_processed = job_info.processed_count - self._logger.debug('--> Job poll check ({}): {}'.format( - stalled_retries, job_info)) if job_info.status in self.JOB_FINAL_STATES: break else: time.sleep(self.BATCH_RETRY_SLEEP_S) - self._logger.debug('--> Job complete: {}'.format(job_info)) - results = self._download_results(request_id) - self._logger.debug('--> Results: {} rows; {}'.format(len(results), results)) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 6031c97..f52d2d7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -99,7 +99,6 @@ class HereMapsGeocoder(Traceable): try: response = self._perform_request(params) result = response['Response']['View'][0]['Result'][0] - self._logger.debug('--> Result: {}'.format(result)) return [self._extract_lng_lat_from_result(result), self._extract_metadata_from_result(result)] except IndexError: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py index 3ddd457..2563d43 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -28,7 +28,6 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: elements = self._encoded_elements(search) - self._logger.debug('--> Sending serial search: {}'.format(search)) result = self.geocode_meta(*elements) results.append((search[0], result[0], result[1])) @@ -52,13 +51,10 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder): free = ', '.join([elem for elem in elements if elem]) frees.append(free) - self._logger.debug('--> sending free search: {}'.format(frees)) full_results = self.geocode_free_text_meta(frees) results = [] - self._logger.debug('--> searches: {}; xy: {}'.format(searches, full_results)) for s, r in zip(searches, full_results): results.append((s[0], r[0], r[1])) - self._logger.debug('--> results: {}'.format(results)) return results def _country_code(self, country): diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 1fc8b4e..0e3cd11 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -53,7 +53,6 @@ class MapboxGeocoder(Traceable): def _parse_geocoder_response(self, response): json_response = json.loads(response) - self._logger.debug('--> json response: {}'.format(json_response)) if json_response: if type(json_response) != list: @@ -154,7 +153,6 @@ class MapboxGeocoder(Traceable): try: free_search = ';'.join([self._escape(fs) for fs in free_searches]) - self._logger.debug('--> free search: {}, country: {}'.format(free_search, country)) response = self._geocoder.forward(address=free_search.decode('utf-8'), country=country) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index c8ad803..8fc2e92 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -37,10 +37,8 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): city = city.encode('utf-8') if city else None state = state.encode('utf-8') if state else None country = country.encode('utf-8') if country else None - self._logger.debug('--> Sending serial search: {}'.format(search)) result = self.geocode_meta(searchtext=address, city=city, state_province=state, country=country) - self._logger.debug('--> result sent') results.append((search_id, result[0], result[1])) return results @@ -50,12 +48,10 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): results = [] for s, r in zip(searches, full_results): results.append((s[0], r[0], r[1])) - self._logger.debug('--> results: {}'.format(results)) return results def _send_batch(self, searches): body = {'batchItems': [{'query': self._query(s)} for s in searches]} - self._logger.debug('--> {}; Body: {}'.format(self.BATCH_URL, body)) request_params = { 'key': self._apikey } @@ -63,9 +59,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): allow_redirects=False, params=request_params, timeout=(self.connect_timeout, self.read_timeout)) - self._logger.debug('--> response: {}'.format(response.status_code)) if response.status_code == 303: - self._logger.debug(response.headers) return response.headers['Location'] else: msg = "Error sending batch: {}; Headers: {}".format( @@ -78,14 +72,12 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): while True: response = self.session.get(self.BASE_URL + location) if response.status_code == 200: - self._logger.debug('--> Results ready {}'.format(location)) return self._parse_results(response.json()) elif response.status_code == 202: stalled_retries += 1 if stalled_retries > self.MAX_STALLED_RETRIES: raise Exception('Too many retries for job {}'.format(location)) location = response.headers['Location'] - self._logger.debug('--> Waiting for {}'.format(location)) time.sleep(self.BATCH_RETRY_SLEEP_S) else: msg = "Error downloading batch: {}; Headers: {}".format( From e9ed3bca18c2c9f37f6c54511e7b1bce1cf413b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 12:27:42 +0200 Subject: [PATCH 129/265] Safer comparison --- .../python/cartodb_services/cartodb_services/mapbox/geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 0e3cd11..a54d06a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -99,7 +99,7 @@ class MapboxGeocoder(Traceable): } def _normalize_relevance(self, relevance): - return 1 if relevance == 0.99 else relevance + return 1 if relevance >= 0.99 else relevance def _validate_input(self, searchtext, city=None, state_province=None, country=None): From c104f6f34bd6e714aa7bafd3eea350a173eb04d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 12:46:16 +0200 Subject: [PATCH 130/265] Metadata attributes constant extraction --- .../python/cartodb_services/cartodb_services/__init__.py | 3 --- .../python/cartodb_services/cartodb_services/geocoder.py | 7 +++++++ .../cartodb_services/cartodb_services/google/geocoder.py | 9 ++++----- .../cartodb_services/here/bulk_geocoder.py | 7 ++++--- .../cartodb_services/cartodb_services/here/geocoder.py | 8 ++++---- .../cartodb_services/cartodb_services/mapbox/geocoder.py | 8 ++++---- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 8 ++++---- test/integration/test_street_functions.py | 1 + 8 files changed, 28 insertions(+), 23 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index dd364b4..ef5caa4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -35,6 +35,3 @@ def _reset(): GD = None from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder - -PRECISION_PRECISE = 'precise' -PRECISION_INTERPOLATED = 'interpolated' diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index c6043e4..47e6d3a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -6,6 +6,13 @@ from collections import namedtuple import json +METADATA_RELEVANCE = 'relevance' +METADATA_PRECISION = 'precision' +METADATA_MATCH_TYPES = 'match_types' + +PRECISION_PRECISE = 'precise' +PRECISION_INTERPOLATED = 'interpolated' + def compose_address(street, city=None, state=None, country=None): return ', '.join(filter(None, [street, city, state, country])) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 44b678c..2f82549 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -4,8 +4,7 @@ from urlparse import parse_qs from exceptions import MalformedResult -from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED -from cartodb_services.geocoder import compose_address +from cartodb_services.geocoder import compose_address, METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory @@ -84,9 +83,9 @@ class GoogleMapsGeocoder(): match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) for match_level in result['types']] return { - 'relevance': base_relevance * partial_factor, - 'precision': PRECISION_BY_LOCATION_TYPE[location_type], - 'match_types': filter(None, match_types) + METADATA_RELEVANCE: base_relevance * partial_factor, + METADATA_PRECISION: PRECISION_BY_LOCATION_TYPE[location_type], + METADATA_MATCH_TYPES: filter(None, match_types) } diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 51b9ec9..b501c6e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -8,6 +8,7 @@ from collections import namedtuple from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder from cartodb_services.here import HereMapsGeocoder +from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException @@ -138,9 +139,9 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']], { - 'relevance': float(row['relevance']), - 'precision': precision, - 'match_types': [match_type] if match_type else [] + METADATA_RELEVANCE: float(row['relevance']), + METADATA_PRECISION: precision, + METADATA_MATCH_TYPES: [match_type] if match_type else [] })) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index f52d2d7..52014b9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,7 +6,7 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * -from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable class HereMapsGeocoder(Traceable): @@ -147,7 +147,7 @@ class HereMapsGeocoder(Traceable): result.get('MatchType', 'pointAddress')] match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) return { - 'relevance': result['Relevance'], - 'precision': precision, - 'match_types': [match_type] if match_type else [] + METADATA_RELEVANCE: result['Relevance'], + METADATA_PRECISION: precision, + METADATA_MATCH_TYPES: [match_type] if match_type else [] } diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index a54d06a..59c707b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -5,7 +5,7 @@ Python client for the Mapbox Geocoder service. import json import requests from mapbox import Geocoder -from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -93,9 +93,9 @@ class MapboxGeocoder(Traceable): match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) for match_level in result['place_type']] return { - 'relevance': self._normalize_relevance(float(result['relevance'])), - 'precision': precision, - 'match_types': filter(None, match_types) + METADATA_RELEVANCE: self._normalize_relevance(float(result['relevance'])), + METADATA_PRECISION: precision, + METADATA_MATCH_TYPES: filter(None, match_types) } def _normalize_relevance(self, relevance): diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 09ed69d..d1c5bed 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -5,7 +5,7 @@ import json import requests from uritemplate import URITemplate from math import tanh -from cartodb_services import PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -145,9 +145,9 @@ class TomTomGeocoder(Traceable): score = self._normalize_score(result['score']) match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['type'], None) return { - 'relevance': score, - 'precision': self._precision_from_score(score), - 'match_types': [match_type] if match_type else [] + METADATA_RELEVANCE: score, + METADATA_PRECISION: self._precision_from_score(score), + METADATA_MATCH_TYPES: [match_type] if match_type else [] } def _normalize_score(self, score): diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 9b134e7..8535139 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -384,6 +384,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): self.metadata['Plaza España, Barcelona'], self.metadata['Santiago Rusiñol 123, Valladolid'] ] + assert_equal(len(response['rows']), len(expected)) for r, e in zip(response['rows'], expected): self.assert_metadata(r['metadata'], e) From d060bd8229bf306c81ed906f341da35039ab826e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 13:24:39 +0200 Subject: [PATCH 131/265] Simplification of batching --- client/cdb_dataservices_client--0.25.0.sql | 12 +++++------- client/sql/21_bulk_geocoding_functions.sql | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index b69a536..b67c219 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -2000,8 +2000,6 @@ DECLARE enough_quota boolean; remaining_quota integer; max_batch_size integer; - max_cartodb_id integer; - min_cartodb_id integer; cartodb_id_batch integer; batches_n integer; @@ -2027,8 +2025,8 @@ BEGIN batch_size := MAX_SAFE_BATCH_SIZE; END IF; - EXECUTE format('SELECT count(1), max(cartodb_id), min(cartodb_id), ceil((max(cartodb_id) + 1 - min(cartodb_id))::float/%s) FROM (%s) _x', batch_size, query) - INTO query_row_count, max_cartodb_id, min_cartodb_id, batches_n; + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; @@ -2058,13 +2056,13 @@ BEGIN 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id - $1)::float/$2) as batch' || + ' floor((row_number() over ())::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $3', street_column, city_column, state_column, country_column, query, temp_table_name) - USING min_cartodb_id, batch_size, cartodb_id_batch; + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 7ab3407..dfbe515 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -6,8 +6,6 @@ DECLARE enough_quota boolean; remaining_quota integer; max_batch_size integer; - max_cartodb_id integer; - min_cartodb_id integer; cartodb_id_batch integer; batches_n integer; @@ -33,8 +31,8 @@ BEGIN batch_size := MAX_SAFE_BATCH_SIZE; END IF; - EXECUTE format('SELECT count(1), max(cartodb_id), min(cartodb_id), ceil((max(cartodb_id) + 1 - min(cartodb_id))::float/%s) FROM (%s) _x', batch_size, query) - INTO query_row_count, max_cartodb_id, min_cartodb_id, batches_n; + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; @@ -64,13 +62,13 @@ BEGIN 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((cartodb_id - $1)::float/$2) as batch' || + ' floor((row_number() over ())::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || 'FROM geocoding_data ' || - 'WHERE batch = $3', street_column, city_column, state_column, country_column, query, temp_table_name) - USING min_cartodb_id, batch_size, cartodb_id_batch; + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; GET DIAGNOSTICS current_row_count = ROW_COUNT; RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; From b90d402fa9e9822c2a791c36458028d9b8127efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 13:53:39 +0200 Subject: [PATCH 132/265] Round relevance (plus refactor) --- .../cartodb_services/cartodb_services/geocoder.py | 12 ++++++++---- .../cartodb_services/google/geocoder.py | 12 ++++++------ .../cartodb_services/here/bulk_geocoder.py | 12 ++++++------ .../cartodb_services/here/geocoder.py | 12 ++++++------ .../cartodb_services/mapbox/geocoder.py | 12 ++++++------ .../cartodb_services/tomtom/geocoder.py | 12 ++++++------ test/integration/test_street_functions.py | 8 ++++---- 7 files changed, 42 insertions(+), 38 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 47e6d3a..465118d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -6,13 +6,17 @@ from collections import namedtuple import json -METADATA_RELEVANCE = 'relevance' -METADATA_PRECISION = 'precision' -METADATA_MATCH_TYPES = 'match_types' - PRECISION_PRECISE = 'precise' PRECISION_INTERPOLATED = 'interpolated' + +def geocoder_metadata(relevance, precision, match_types): + return { + 'relevance': round(relevance, 2), + 'precision': precision, + 'match_types': match_types + } + def compose_address(street, city=None, state=None, country=None): return ', '.join(filter(None, [street, city, state, country])) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index 2f82549..fb75c76 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -4,7 +4,7 @@ from urlparse import parse_qs from exceptions import MalformedResult -from cartodb_services.geocoder import compose_address, METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory @@ -82,11 +82,11 @@ class GoogleMapsGeocoder(): partial_factor = PARTIAL_FACTOR if partial_match else 1 match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) for match_level in result['types']] - return { - METADATA_RELEVANCE: base_relevance * partial_factor, - METADATA_PRECISION: PRECISION_BY_LOCATION_TYPE[location_type], - METADATA_MATCH_TYPES: filter(None, match_types) - } + return geocoder_metadata( + base_relevance * partial_factor, + PRECISION_BY_LOCATION_TYPE[location_type], + filter(None, match_types) + ) def _build_optional_parameters(self, city=None, state=None, diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index b501c6e..8316c44 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -8,7 +8,7 @@ from collections import namedtuple from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder from cartodb_services.here import HereMapsGeocoder -from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES +from cartodb_services.geocoder import geocoder_metadata from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException @@ -138,11 +138,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None) results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']], - { - METADATA_RELEVANCE: float(row['relevance']), - METADATA_PRECISION: precision, - METADATA_MATCH_TYPES: [match_type] if match_type else [] - })) + geocoder_metadata( + float(row['relevance']), + precision, + [match_type] if match_type else [] + ))) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 52014b9..23dd871 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,7 +6,7 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * -from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata from cartodb_services.metrics import Traceable class HereMapsGeocoder(Traceable): @@ -146,8 +146,8 @@ class HereMapsGeocoder(Traceable): precision = self.PRECISION_BY_MATCH_TYPE[ result.get('MatchType', 'pointAddress')] match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) - return { - METADATA_RELEVANCE: result['Relevance'], - METADATA_PRECISION: precision, - METADATA_MATCH_TYPES: [match_type] if match_type else [] - } + return geocoder_metadata( + result['Relevance'], + precision, + [match_type] if match_type else [] + ) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 59c707b..d76937a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -5,7 +5,7 @@ Python client for the Mapbox Geocoder service. import json import requests from mapbox import Geocoder -from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -92,11 +92,11 @@ class MapboxGeocoder(Traceable): match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None) for match_level in result['place_type']] - return { - METADATA_RELEVANCE: self._normalize_relevance(float(result['relevance'])), - METADATA_PRECISION: precision, - METADATA_MATCH_TYPES: filter(None, match_types) - } + return geocoder_metadata( + self._normalize_relevance(float(result['relevance'])), + precision, + filter(None, match_types) + ) def _normalize_relevance(self, relevance): return 1 if relevance >= 0.99 else relevance diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index d1c5bed..46cdbda 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -5,7 +5,7 @@ import json import requests from uritemplate import URITemplate from math import tanh -from cartodb_services.geocoder import METADATA_RELEVANCE, METADATA_PRECISION, METADATA_MATCH_TYPES, PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -144,11 +144,11 @@ class TomTomGeocoder(Traceable): def _extract_metadata_from_result(self, result): score = self._normalize_score(result['score']) match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['type'], None) - return { - METADATA_RELEVANCE: score, - METADATA_PRECISION: self._precision_from_score(score), - METADATA_MATCH_TYPES: [match_type] if match_type else [] - } + return geocoder_metadata( + score, + self._precision_from_score(score), + [match_type] if match_type else [] + ) def _normalize_score(self, score): return tanh(score * SCORE_NORMALIZATION_FACTOR) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 8535139..df30d66 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -87,14 +87,14 @@ class TestStreetFunctionsSetUp(TestCase): 'Plaza España, Barcelona': {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.448283494533, 'precision': 'interpolated', 'match_types': ['street']} + {'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']} } MAPBOX_METADATAS = { 'Plaza España, Barcelona': - {'relevance': 0.666, 'precision': 'precise', 'match_types': ['point_of_interest']}, + {'relevance': 0.67, 'precision': 'precise', 'match_types': ['point_of_interest']}, 'Santiago Rusiñol 123, Valladolid': - {'relevance': 0.666, 'precision': 'precise', 'match_types': ['point_of_interest']} # TODO: wrong + {'relevance': 0.67, 'precision': 'precise', 'match_types': ['point_of_interest']} # TODO: wrong } METADATAS = { @@ -410,7 +410,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): def assert_metadata(metadata, expected): relevance = metadata['relevance'] expected_relevance = expected['relevance'] - assert_true(isclose(relevance, expected_relevance, 0.05), + assert_true(isclose(relevance, expected_relevance, 0.02), '{} not close to {}'.format(relevance, expected_relevance)) assert_equal(metadata['precision'], expected['precision']) From 5e8dbaf2399eff58460edcf8856a757170300f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 13:56:44 +0200 Subject: [PATCH 133/265] Improvements in fixtures accuracy --- test/integration/test_street_functions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index df30d66..a847637 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -37,7 +37,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Madrid': [-3.70578, 40.42028], 'Logroño, Spain': [-2.45194, 42.46592], 'Logroño, Argentina': [-61.69604, -29.50425], - 'Plaza España, Barcelona': [2.1735699, 41.3823] # TODO: not ideal + 'Plaza España, Barcelona': [2.14834, 41.37494] } TOMTOM_POINTS = HERE_POINTS.copy() @@ -48,7 +48,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Valladolid, Spain': [-4.72838, 41.6542], 'Madrid': [-3.70035, 40.42028], 'Logroño, Spain': [-2.44998, 42.46592], - 'Plaza España, Barcelona': [2.07479, 41.36818] # TODO: not ideal + 'Plaza España, Barcelona': [2.1497, 41.37516] }) MAPBOX_POINTS = GOOGLE_POINTS.copy() @@ -59,7 +59,7 @@ class TestStreetFunctionsSetUp(TestCase): 'Valladolid, Spain': [-4.72856, 41.652251], '1902 amphitheatre parkway': [-118.03, 34.06], # TODO: huge mismatch 'Madrid': [-3.69194, 40.4167754], - 'Plaza España, Barcelona': [2.245969, 41.452483] # TODO: not ideal + 'Plaza España, Barcelona': [2.342231, 41.50677] # TODO: not ideal }) FIXTURE_POINTS = { @@ -341,7 +341,6 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): x_y_by_cartodb_id = self._x_y_by_cartodb_id(response) assert_equal(x_y_by_cartodb_id[1], x_y_by_cartodb_id[2]) - # "'Plaza España', 'Barcelona', null, 'Spain') as the_geom) _x" def test_component_aggregation(self): query = "select cartodb_id, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ From 5e34faefe55ce80d0fe91d9938f92e96f58f7559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 14:39:24 +0200 Subject: [PATCH 134/265] Quota test --- .../python/cartodb_services/cartodb_services/geocoder.py | 1 + test/integration/test_street_functions.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 465118d..54e80f1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -17,6 +17,7 @@ def geocoder_metadata(relevance, precision, match_types): 'match_types': match_types } + def compose_address(street, city=None, state=None, country=None): return ', '.join(filter(None, [street, city, state, country])) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index a847637..ebcc26f 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -131,6 +131,11 @@ class TestStreetFunctionsSetUp(TestCase): return IntegrationTestHelper.execute_query_raw(self.sql_api_url, authenticated_query) + def _used_quota(self): + query = "select used_quota " \ + "from cdb_dataservices_client.cdb_service_quota_info() " \ + "where service = 'hires_geocoder'" + return self._run_authenticated(query)['rows'][0]['used_quota'] class TestStreetFunctions(TestStreetFunctionsSetUp): @@ -305,6 +310,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): 'Toronto, Canada"}}'.format(first_cartodb_id + i, first_street_number + i)) + used_quota = self._used_quota() + query = "select *, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ "'select * from jsonb_to_recordset(''[" \ @@ -321,6 +328,8 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_not_equal(metadata['precision'], None) assert_not_equal(metadata['match_types'], None) + assert_equal(self._used_quota(), used_quota + n) + def test_missing_components_on_private_function(self): query = "SELECT _cdb_bulk_geocode_street_point(" \ " '[{\"id\": \"1\", \"address\": \"Amphitheatre Parkway 22\"}]' " \ From 0c5e9da02817154ebdd1c8c87b59a2e895546ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 17 Jul 2018 19:48:13 +0200 Subject: [PATCH 135/265] Upgrade/downgrade scripts for server 0.32.0 and client 0.25.0 --- ...db_dataservices_client--0.24.0--0.25.0.sql | 193 ++++++++++++++++++ ...db_dataservices_client--0.25.0--0.24.0.sql | 25 +++ ...db_dataservices_client--0.23.0--0.24.0.sql | 0 ...db_dataservices_client--0.24.0--0.23.0.sql | 0 .../cdb_dataservices_client--0.24.0.sql | 0 ...db_dataservices_server--0.31.0--0.32.0.sql | 176 ++++++++++++++++ ...db_dataservices_server--0.32.0--0.31.0.sql | 78 +++++++ ...db_dataservices_server--0.30.5--0.31.0.sql | 0 ...db_dataservices_server--0.31.0--0.30.5.sql | 0 .../cdb_dataservices_server--0.31.0.sql | 0 10 files changed, 472 insertions(+) create mode 100644 client/cdb_dataservices_client--0.24.0--0.25.0.sql create mode 100644 client/cdb_dataservices_client--0.25.0--0.24.0.sql rename client/{ => old_versions}/cdb_dataservices_client--0.23.0--0.24.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.24.0--0.23.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.24.0.sql (100%) create mode 100644 server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.30.5--0.31.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.31.0--0.30.5.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.31.0.sql (100%) diff --git a/client/cdb_dataservices_client--0.24.0--0.25.0.sql b/client/cdb_dataservices_client--0.24.0--0.25.0.sql new file mode 100644 index 0000000..2bebf96 --- /dev/null +++ b/client/cdb_dataservices_client--0.24.0--0.25.0.sql @@ -0,0 +1,193 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE;-- + + +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +ALTER TYPE cdb_dataservices_client.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + current_row_count integer ; + + temp_table_name text; +BEGIN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over ())::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + GET DIAGNOSTICS current_row_count = ROW_COUNT; + RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0--0.24.0.sql b/client/cdb_dataservices_client--0.25.0--0.24.0.sql new file mode 100644 index 0000000..98816a6 --- /dev/null +++ b/client/cdb_dataservices_client--0.25.0--0.24.0.sql @@ -0,0 +1,25 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS count_estimate(query text); + +DROP FUNCTION IF EXISTS jsonb_array_casttext(jsonb); + +ALTER TYPE cdb_dataservices_client.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, + text, text, text, text, integer); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb); + +DROP TYPE IF EXISTS cdb_dataservices_client.geocoding; + diff --git a/client/cdb_dataservices_client--0.23.0--0.24.0.sql b/client/old_versions/cdb_dataservices_client--0.23.0--0.24.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.23.0--0.24.0.sql rename to client/old_versions/cdb_dataservices_client--0.23.0--0.24.0.sql diff --git a/client/cdb_dataservices_client--0.24.0--0.23.0.sql b/client/old_versions/cdb_dataservices_client--0.24.0--0.23.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.24.0--0.23.0.sql rename to client/old_versions/cdb_dataservices_client--0.24.0--0.23.0.sql diff --git a/client/cdb_dataservices_client--0.24.0.sql b/client/old_versions/cdb_dataservices_client--0.24.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.24.0.sql rename to client/old_versions/cdb_dataservices_client--0.24.0.sql diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql new file mode 100644 index 0000000..9e7687c --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql @@ -0,0 +1,176 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +ALTER TYPE cdb_dataservices_server.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + result = plpy.execute(plan, [username, orgname, searches]) + return result + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql new file mode 100644 index 0000000..549daca --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql @@ -0,0 +1,78 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +ALTER TYPE cdb_dataservices_server.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocode_street_point(TEXT, TEXT, jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_google_geocode_street_point(TEXT, TEXT, jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(TEXT, TEXT, jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(TEXT, TEXT, jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(TEXT, TEXT, jsonb); +DROP TYPE IF EXISTS cdb_dataservices_server.geocoding; diff --git a/server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.30.5--0.31.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.30.5--0.31.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.30.5--0.31.0.sql diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql b/server/extension/old_versions/cdb_dataservices_server--0.31.0--0.30.5.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.31.0--0.30.5.sql rename to server/extension/old_versions/cdb_dataservices_server--0.31.0--0.30.5.sql diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.31.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.31.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.31.0.sql From b0c1948c14c015c607555ce586b1f6301d7679ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 18 Jul 2018 14:29:50 +0200 Subject: [PATCH 136/265] TL;DR: safer deployment and minor fixes - Instead of modifying cdb_service_quota_info to return max_batch_size, a new type (service_quota_info_batch) and a new function (cdb_service_quota_info_batch) are created. That makes deployment safe. - Fixes geocoding with forced batch size 1. - Improves namespacing for count_estimate (-> cdb_dataservices_client.cdb_count_estimate). - Improves namespacing for jsonb_array_casttext (-> cdb_dataservices_client.cdb_jsonb_array_casttext). --- ...db_dataservices_client--0.24.0--0.25.0.sql | 106 ++++++++++++++++-- ...db_dataservices_client--0.25.0--0.24.0.sql | 18 +-- client/cdb_dataservices_client--0.25.0.sql | 93 +++++++++++++-- client/renderer/interface.yaml | 7 ++ client/sql/05_utils.sql | 4 +- client/sql/16_custom_types.sql | 8 ++ client/sql/21_bulk_geocoding_functions.sql | 9 +- .../21_bulk_geocoding_functions_test.out | 10 +- .../sql/21_bulk_geocoding_functions_test.sql | 10 +- ...db_dataservices_server--0.31.0--0.32.0.sql | 94 ++++++---------- ...db_dataservices_server--0.32.0--0.31.0.sql | 67 +---------- .../cdb_dataservices_server--0.32.0.sql | 62 ++++++++-- server/extension/sql/200_quotas.sql | 59 ++++++++-- .../extension/sql/21_bulk_geocode_street.sql | 3 +- .../cartodb_services/tools/log.py | 4 + test/integration/test_street_functions.py | 18 +++ 16 files changed, 375 insertions(+), 197 deletions(-) diff --git a/client/cdb_dataservices_client--0.24.0--0.25.0.sql b/client/cdb_dataservices_client--0.24.0--0.25.0.sql index 2bebf96..fd742f5 100644 --- a/client/cdb_dataservices_client--0.24.0--0.25.0.sql +++ b/client/cdb_dataservices_client--0.24.0--0.25.0.sql @@ -7,7 +7,7 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -23,18 +23,30 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE;-- - CREATE TYPE cdb_dataservices_client.geocoding AS ( cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb ); -ALTER TYPE cdb_dataservices_client.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC +); + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ @@ -55,12 +67,33 @@ BEGIN RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + -- -- Public dataservices API function -- -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ @@ -74,13 +107,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -120,12 +152,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -133,9 +164,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; @@ -143,6 +171,10 @@ BEGIN END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -175,19 +207,69 @@ BEGIN END; END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + -- -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ CONNECT cdb_dataservices_client._server_conn_str(); - + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0--0.24.0.sql b/client/cdb_dataservices_client--0.25.0--0.24.0.sql index 98816a6..15b4950 100644 --- a/client/cdb_dataservices_client--0.25.0--0.24.0.sql +++ b/client/cdb_dataservices_client--0.25.0--0.24.0.sql @@ -6,20 +6,24 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -DROP FUNCTION IF EXISTS count_estimate(query text); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_count_estimate(query text); -DROP FUNCTION IF EXISTS jsonb_array_casttext(jsonb); - -ALTER TYPE cdb_dataservices_client.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb); -DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, - text, text, text, text, integer); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_quota_info_batch(); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, text, text, text, text, integer); DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe (); + DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb); -DROP TYPE IF EXISTS cdb_dataservices_client.geocoding; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (text, text); +DROP TYPE IF EXISTS cdb_dataservices_client.service_quota_info_batch; + +DROP TYPE IF EXISTS cdb_dataservices_client.geocoding; diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index b67c219..5ef3955 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -5,7 +5,7 @@ -- Make sure we have a sane search path to create/update the extension SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -21,7 +21,7 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE;-- -- Geocoder server connection config @@ -113,6 +113,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( ); CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( service cdb_dataservices_client.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -1601,6 +1609,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN AS $$ DECLARE @@ -2005,13 +2038,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -2051,12 +2083,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -2064,9 +2095,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; @@ -4201,6 +4229,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN AS $$ DECLARE @@ -4845,6 +4909,14 @@ RETURNS SETOF service_quota_info AS $$ SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) @@ -5126,6 +5198,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exc GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 394a5fa..54805cd 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -517,6 +517,13 @@ params: - {} +- name: cdb_service_quota_info_batch + return_type: SETOF service_quota_info_batch + multi_row: true + multi_field: true + params: + - {} + - name: cdb_enough_quota return_type: BOOLEAN params: diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index 31c8445..c245422 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -1,5 +1,5 @@ -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -15,6 +15,6 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE; \ No newline at end of file diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 7458ffa..15dc782 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -35,6 +35,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( ); CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( service cdb_dataservices_client.service_type, monthly_quota NUMERIC, used_quota NUMERIC, diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index dfbe515..5af493b 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -11,13 +11,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -57,12 +56,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -70,9 +68,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index d2f268b..c846a1d 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -1,7 +1,7 @@ \set VERBOSITY terse -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$ SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; $$ LANGUAGE SQL; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; @@ -15,7 +15,7 @@ ERROR: Remaining quota: 0. Estimated cost: 1 -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Remaining quota: 0. Estimated cost: 1 -DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index d17f470..985133a 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -1,8 +1,8 @@ \set VERBOSITY terse -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$ SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; $$ LANGUAGE SQL; @@ -18,9 +18,9 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql index 9e7687c..45faaac 100644 --- a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql @@ -4,75 +4,48 @@ -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( username TEXT, orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ - from cartodb_services.metrics.user import UserMetricsService - from datetime import date +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - today = date.today() ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 - #-- Isolines - service = 'isolines' - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] - - #-- Hires Geocoder - service = 'hires_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] - - #-- Routing - service = 'routing' - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_routing_config, redis_conn) - - monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config.service_type, today) - soft_limit = user_routing_config.soft_limit - provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] - - #-- Observatory - service = 'observatory' - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - user_service = UserMetricsService(user_obs_config, redis_conn) - - monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config.service_type, today) - soft_limit = user_obs_config.soft_limit - provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -114,8 +87,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql index 549daca..2609e9d 100644 --- a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql @@ -4,75 +4,12 @@ -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( - username TEXT, - orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ - from cartodb_services.metrics.user import UserMetricsService - from datetime import date - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - today = date.today() - ret = [] - - #-- Isolines - service = 'isolines' - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) - - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Hires Geocoder - service = 'hires_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Routing - service = 'routing' - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_routing_config, redis_conn) - - monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config.service_type, today) - soft_limit = user_routing_config.soft_limit - provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Observatory - service = 'observatory' - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - user_service = UserMetricsService(user_obs_config, redis_conn) - - monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config.service_type, today) - soft_limit = user_obs_config.soft_limit - provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_quota_info_batch(TEXT, TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_google_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(TEXT, TEXT, jsonb); DROP TYPE IF EXISTS cdb_dataservices_server.geocoding; +DROP TYPE IF EXISTS cdb_dataservices_server.service_quota_info_batch; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index dc82322..42d9b9e 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -1857,6 +1857,21 @@ BEGIN WHERE pg_type.typname = 'service_quota_info' AND pg_namespace.nspname = 'cdb_dataservices_server') THEN CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -1867,13 +1882,13 @@ BEGIN END IF; END $$; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( username TEXT, orgname TEXT) RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date - from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -1891,7 +1906,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Hires Geocoder service = 'hires_geocoder' @@ -1903,12 +1918,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Routing service = 'routing' @@ -1920,7 +1930,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Observatory service = 'observatory' @@ -1932,12 +1942,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, @@ -2385,8 +2424,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 6f34083..973d1ce 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -18,6 +18,21 @@ BEGIN WHERE pg_type.typname = 'service_quota_info' AND pg_namespace.nspname = 'cdb_dataservices_server') THEN CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -28,13 +43,13 @@ BEGIN END IF; END $$; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( username TEXT, orgname TEXT) RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date - from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -52,7 +67,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Hires Geocoder service = 'hires_geocoder' @@ -64,12 +79,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Routing service = 'routing' @@ -81,7 +91,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Observatory service = 'observatory' @@ -93,12 +103,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 309d323..cee0539 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -35,8 +35,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index 500c8ab..f38febb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -94,7 +94,11 @@ class Logger: else: exception_message = '' + # Adding trace breaks tests + # trace = traceback.format_exc(15) + # message = '{}{}. Trace: {}'.format(text, exception_message, trace) message = '{}{}'.format(text, exception_message) + if self._check_plpy(): if level == 'debug': plpy.debug(message) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index ebcc26f..2d02bdb 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -230,6 +230,24 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): } self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + def test_batch_size_1(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1900 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 2, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 3, \"address\": \"1902 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "'address', null, null, null, 1)" + response = self._run_authenticated(query) + + points_by_cartodb_id = { + 1: self.fixture_points['1900 amphitheatre parkway'], + 2: self.fixture_points['1901 amphitheatre parkway'], + 3: self.fixture_points['1902 amphitheatre parkway'], + } + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + def test_city_column_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ From 5c09a2eb29a48a3fd4f0065cb6dcb0d336de0ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 10:36:49 +0200 Subject: [PATCH 137/265] Missing grant --- client/cdb_dataservices_client--0.24.0--0.25.0.sql | 3 ++- client/cdb_dataservices_client--0.25.0.sql | 5 ++++- client/sql/95_grant_execute_manual.sql | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/cdb_dataservices_client--0.24.0--0.25.0.sql b/client/cdb_dataservices_client--0.24.0--0.25.0.sql index fd742f5..1f37285 100644 --- a/client/cdb_dataservices_client--0.24.0--0.25.0.sql +++ b/client/cdb_dataservices_client--0.24.0--0.25.0.sql @@ -7,7 +7,7 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -273,3 +273,4 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point( GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 5ef3955..608684b 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -5212,4 +5212,7 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_ex GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; \ No newline at end of file +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/sql/95_grant_execute_manual.sql b/client/sql/95_grant_execute_manual.sql index 8d6a013..099c8df 100644 --- a/client/sql/95_grant_execute_manual.sql +++ b/client/sql/95_grant_execute_manual.sql @@ -1,4 +1,7 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; \ No newline at end of file +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; From 887fc1591529c523c45863297c41f3046437b384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 11:31:35 +0200 Subject: [PATCH 138/265] NEWS versions update --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 786d41d..9883417 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +Jul 19th, 2018 +============== +* Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.0` of the Python library. + * Support for batch street-level geocoding. + May 7th, 2018 ============= * Version `0.24.0` of the client, `0.31.0` of the server, and `0.18.0` of the python library. From 0d490bbb19d110e19d0a53fd36c05b10208fbf94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 20:52:59 +0200 Subject: [PATCH 139/265] Extract EMPTY_RESPONSE --- .../lib/python/cartodb_services/cartodb_services/geocoder.py | 2 ++ .../cartodb_services/cartodb_services/google/geocoder.py | 3 +-- .../cartodb_services/cartodb_services/mapbox/geocoder.py | 5 ++--- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 54e80f1..5a9f0a6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -9,6 +9,8 @@ import json PRECISION_PRECISE = 'precise' PRECISION_INTERPOLATED = 'interpolated' +EMPTY_RESPONSE = [[], {}] + def geocoder_metadata(relevance, precision, match_types): return { diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index fb75c76..30a62cf 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -4,11 +4,10 @@ from urlparse import parse_qs from exceptions import MalformedResult -from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED +from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE from cartodb_services.google.exceptions import InvalidGoogleCredentials from client_factory import GoogleMapsClientFactory -EMPTY_RESPONSE = [[], {}] PARTIAL_FACTOR = 0.8 RELEVANCE_BY_LOCATION_TYPE = { 'ROOFTOP': 1, diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index d76937a..2af2c6f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -5,7 +5,7 @@ Python client for the Mapbox Geocoder service. import json import requests from mapbox import Geocoder -from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -23,8 +23,6 @@ ENTRY_COORDINATES = 'coordinates' ENTRY_TYPE = 'type' TYPE_POINT = 'Point' -EMPTY_RESPONSE = [[], {}] - MATCH_TYPE_BY_MATCH_LEVEL = { 'poi': 'point_of_interest', 'poi.landmark': 'point_of_interest', @@ -154,6 +152,7 @@ class MapboxGeocoder(Traceable): try: free_search = ';'.join([self._escape(fs) for fs in free_searches]) response = self._geocoder.forward(address=free_search.decode('utf-8'), + limit=1, country=country) if response.status_code == requests.codes.ok: diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 46cdbda..3f077bd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -5,7 +5,7 @@ import json import requests from uritemplate import URITemplate from math import tanh -from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -20,7 +20,6 @@ ENTRY_RESULTS = 'results' ENTRY_POSITION = 'position' ENTRY_LON = 'lon' ENTRY_LAT = 'lat' -EMPTY_RESPONSE = [[], {}] SCORE_NORMALIZATION_FACTOR = 0.15 PRECISION_SCORE_THRESHOLD = 0.5 From 96fbf3080a23f5a91b8f0948d1a58d54ab979e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 21:08:07 +0200 Subject: [PATCH 140/265] Base run_street_point_geocoder test --- .../test/test_cartodb_services_geocoder.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py new file mode 100644 index 0000000..a23554c --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +from unittest import TestCase +from mock import Mock, MagicMock +from nose.tools import assert_not_equal, assert_equal, assert_true +from cartodb_services.geocoder import run_street_point_geocoder + + +class TestRunStreetPointGeocoder(TestCase): + def test_count_increment_total_service_use_on_error(self): + quota_service_mock = Mock() + + service_manager_mock = Mock() + service_manager_mock.quota_service = quota_service_mock + + searches = [] + + logger_config_mock = MagicMock(min_log_level='debug', + log_file_path='/tmp/ptest.log') + with(self.assertRaises(BaseException)): + run_street_point_geocoder(Mock(), + {'logger_config': logger_config_mock}, + None, + service_manager_mock, + None, None, searches) + + quota_service_mock.increment_total_service_use. \ + assert_called_once_with() From fd097724f11a127f488ecdffb253fea2fade1d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 21:17:09 +0200 Subject: [PATCH 141/265] In case of general error, total service use should be incremented by searches length --- server/lib/python/cartodb_services/cartodb_services/geocoder.py | 2 +- .../cartodb_services/test/test_cartodb_services_geocoder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 5a9f0a6..2f1b55a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -62,7 +62,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org service_manager.logger.error('Error trying to bulk geocode street point', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to bulk geocode street') finally: - service_manager.quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use(len(searches)) StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state country') diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index a23554c..ea3deac 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -26,4 +26,4 @@ class TestRunStreetPointGeocoder(TestCase): None, None, searches) quota_service_mock.increment_total_service_use. \ - assert_called_once_with() + assert_called_once_with(len(searches)) From cd5e6510a68f4740fba1743bc787edb59fb0e3be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 19 Jul 2018 21:19:00 +0200 Subject: [PATCH 142/265] In case of general error, failed service use should be incremented by searches length --- .../python/cartodb_services/cartodb_services/geocoder.py | 4 +++- .../test/test_cartodb_services_geocoder.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 2f1b55a..1678552 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -54,11 +54,13 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org service_manager.quota_service.increment_empty_service_use(len(searches)) return [] except QuotaExceededException as qe: + logger.debug('QuotaExceededException at run_street_point_geocoder', qe, + data={"username": username, "orgname": orgname}) service_manager.quota_service.increment_failed_service_use(len(searches)) return [] except BaseException as e: import sys - service_manager.quota_service.increment_failed_service_use() + service_manager.quota_service.increment_failed_service_use(len(searches)) service_manager.logger.error('Error trying to bulk geocode street point', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to bulk geocode street') finally: diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index ea3deac..044dca0 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -8,11 +8,13 @@ from cartodb_services.geocoder import run_street_point_geocoder class TestRunStreetPointGeocoder(TestCase): - def test_count_increment_total_service_use_on_error(self): + def test_count_increment_total_and_failed_service_use_on_error(self): quota_service_mock = Mock() service_manager_mock = Mock() service_manager_mock.quota_service = quota_service_mock + service_manager_mock.assert_within_limits = \ + Mock(side_effect=Exception('Fail!')) searches = [] @@ -27,3 +29,6 @@ class TestRunStreetPointGeocoder(TestCase): quota_service_mock.increment_total_service_use. \ assert_called_once_with(len(searches)) + + quota_service_mock.increment_failed_service_use. \ + assert_called_once_with(len(searches)) From abbaf83e9769bd40711a3d67c21a719294919931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 10:28:38 +0200 Subject: [PATCH 143/265] run_street_point_geocoder tests --- .../test/test_cartodb_services_geocoder.py | 63 ++++++++++++++----- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index 044dca0..4670fe6 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -4,31 +4,64 @@ from unittest import TestCase from mock import Mock, MagicMock from nose.tools import assert_not_equal, assert_equal, assert_true -from cartodb_services.geocoder import run_street_point_geocoder +from cartodb_services.geocoder import run_street_point_geocoder, StreetGeocoderSearch class TestRunStreetPointGeocoder(TestCase): - def test_count_increment_total_and_failed_service_use_on_error(self): - quota_service_mock = Mock() - service_manager_mock = Mock() - service_manager_mock.quota_service = quota_service_mock - service_manager_mock.assert_within_limits = \ + def setUp(self): + point = [0,0] + self.plpy_mock = Mock() + self.plpy_mock.execute = MagicMock(return_value=[{'the_geom': point}]) + + self.logger_config_mock = MagicMock(min_log_level='debug', + log_file_path='/tmp/ptest.log') + self.gd_mock = {'logger_config': self.logger_config_mock} + + self.geocoder_mock = Mock() + + self.quota_service_mock = Mock() + + self.service_manager_mock = Mock() + self.service_manager_mock.quota_service = self.quota_service_mock + self.service_manager_mock.assert_within_limits = MagicMock() + + def test_count_increment_total_and_failed_service_use_on_error(self): + self.service_manager_mock.assert_within_limits = \ Mock(side_effect=Exception('Fail!')) searches = [] - logger_config_mock = MagicMock(min_log_level='debug', - log_file_path='/tmp/ptest.log') with(self.assertRaises(BaseException)): - run_street_point_geocoder(Mock(), - {'logger_config': logger_config_mock}, - None, - service_manager_mock, - None, None, searches) + run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) - quota_service_mock.increment_total_service_use. \ + self.quota_service_mock.increment_total_service_use. \ assert_called_once_with(len(searches)) - quota_service_mock.increment_failed_service_use. \ + self.quota_service_mock.increment_failed_service_use. \ assert_called_once_with(len(searches)) + + def test_increment_success_service_use_on_complete_response(self): + searches = [ + StreetGeocoderSearch(id=1, address='Paseo Zorrilla 1, Valladolid', + city=None, state=None, country=None), + StreetGeocoderSearch(id=2, address='Paseo Zorrilla 2, Valladolid', + city=None, state=None, country=None) + ] + results = [ + (1, [0, 0], {}), + (2, [0, 0], {}), + ] + self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) + run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + + self.quota_service_mock.increment_success_service_use.\ + assert_called_once_with(len(results)) + + From 2862c8002530c9b291bd9bbad67efa94e9f71833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 11:53:02 +0200 Subject: [PATCH 144/265] Proper empty count on bulk geocoding --- .../cartodb_services/geocoder.py | 17 +-- .../test/test_cartodb_services_geocoder.py | 102 ++++++++++++++++-- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 1678552..edb33bc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -30,11 +30,13 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org logger = Logger(logger_config) + success_count, failed_count = 0, 0 + try: service_manager.assert_within_limits(quota=False) geocode_results = geocoder.bulk_geocode(searches=searches) + results = [] if geocode_results: - results = [] for result in geocode_results: if len(result) > 2: metadata = json.dumps(result[2]) @@ -46,13 +48,16 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] results.append([result[0], point['the_geom'], metadata]) + success_count += 1 else: results.append([result[0], None, metadata]) - service_manager.quota_service.increment_success_service_use(len(results)) - return results - else: - service_manager.quota_service.increment_empty_service_use(len(searches)) - return [] + + empty_count = len(searches) - success_count - failed_count + service_manager.quota_service.increment_success_service_use(success_count) + service_manager.quota_service.increment_empty_service_use(empty_count) + service_manager.quota_service.increment_failed_service_use(failed_count) + + return results except QuotaExceededException as qe: logger.debug('QuotaExceededException at run_street_point_geocoder', qe, data={"username": username, "orgname": orgname}) diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index 4670fe6..240e6f7 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -4,9 +4,44 @@ from unittest import TestCase from mock import Mock, MagicMock from nose.tools import assert_not_equal, assert_equal, assert_true +from cartodb_services.tools import QuotaExceededException from cartodb_services.geocoder import run_street_point_geocoder, StreetGeocoderSearch +SEARCH_FIXTURES = { + 'two': [ + StreetGeocoderSearch(id=1, address='Paseo Zorrilla 1, Valladolid', + city=None, state=None, country=None), + StreetGeocoderSearch(id=2, address='Paseo Zorrilla 2, Valladolid', + city=None, state=None, country=None) + ], + 'wrong': [ + StreetGeocoderSearch(id=100, address='deowpfjoepwjfopejwpofjewpojgf', + city=None, state=None, country=None), + ] +} + +BULK_RESULTS_FIXTURES = { + 'two': [ + (1, [0, 0], {}), + (2, [0, 0], {}), + ], + 'wrong': [ + (100, [], {}) + ] +} + +EXPECTED_RESULTS_FIXTURES = { + 'two': [ + [1, [0, 0], '{}'], + [2, [0, 0], '{}'], + ], + 'wrong': [ + [100, None, '{}'] + ] +} + + class TestRunStreetPointGeocoder(TestCase): def setUp(self): @@ -44,24 +79,69 @@ class TestRunStreetPointGeocoder(TestCase): self.quota_service_mock.increment_failed_service_use. \ assert_called_once_with(len(searches)) + def test_count_increment_failed_service_use_on_quota_error(self): + self.service_manager_mock.assert_within_limits = \ + Mock(side_effect=QuotaExceededException()) + + searches = SEARCH_FIXTURES['two'] + + result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + assert_equal(result, []) + + self.quota_service_mock.increment_failed_service_use. \ + assert_called_once_with(len(searches)) + def test_increment_success_service_use_on_complete_response(self): - searches = [ - StreetGeocoderSearch(id=1, address='Paseo Zorrilla 1, Valladolid', - city=None, state=None, country=None), - StreetGeocoderSearch(id=2, address='Paseo Zorrilla 2, Valladolid', - city=None, state=None, country=None) - ] + searches = SEARCH_FIXTURES['two'] results = [ (1, [0, 0], {}), (2, [0, 0], {}), ] + expected_results = [ + [1, [0, 0], '{}'], + [2, [0, 0], '{}'], + ] self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) - run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + assert_equal(result, expected_results) - self.quota_service_mock.increment_success_service_use.\ + self.quota_service_mock.increment_success_service_use. \ assert_called_once_with(len(results)) + def test_increment_empty_service_use_on_complete_response(self): + searches = SEARCH_FIXTURES['two'] + results = [] + self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) + result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + + assert_equal(result, results) + + self.quota_service_mock.increment_empty_service_use. \ + assert_called_once_with(len(searches)) + + def test_increment_mixed_empty_service_use_on_complete_response(self): + searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['wrong'] + bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['wrong'] + self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) + result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + + assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['wrong']) + + self.quota_service_mock.increment_success_service_use. \ + assert_called_once_with(len(SEARCH_FIXTURES['two'])) + self.quota_service_mock.increment_empty_service_use. \ + assert_called_once_with(len(SEARCH_FIXTURES['wrong'])) + From 1cebbe7af0946ca8a6182fe3e7cc29514a2fd018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 12:16:48 +0200 Subject: [PATCH 145/265] Missing warning mock and fix for debug --- server/lib/python/cartodb_services/test/mock_plpy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/mock_plpy.py b/server/lib/python/cartodb_services/test/mock_plpy.py index 0fe067f..c088670 100644 --- a/server/lib/python/cartodb_services/test/mock_plpy.py +++ b/server/lib/python/cartodb_services/test/mock_plpy.py @@ -52,11 +52,14 @@ class MockPlPy: self._logged_queries = [] self._log_executed_queries = True + def warning(self, msg): + self.warnings.append(msg) + def notice(self, msg): self.notices.append(msg) def debug(self, msg): - self.notices.append(msg) + self.debugs.append(msg) def info(self, msg): self.infos.append(msg) From 9a1b1e2832030a91704c0dafb9e1fcc89f339ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 12:18:26 +0200 Subject: [PATCH 146/265] Error count --- .../cartodb_services/geocoder.py | 18 ++++++++---- .../test/test_cartodb_services_geocoder.py | 29 ++++++++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index edb33bc..8d5d456 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -39,18 +39,24 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org if geocode_results: for result in geocode_results: if len(result) > 2: - metadata = json.dumps(result[2]) + metadata = result[2] else: - logger.warning('Geocoding for {} without metadata'.format(username)) - metadata = '{}' + logger.warning('Geocoding without metadata', + data={"username": username, "orgname": orgname}) + metadata = {} - if result[1] and len(result[1]) == 2: + if metadata.get('error', None): + logger.warning('Geocoding error', + data={"username": username, "orgname": orgname, "error": metadata['error']}) + results.append([result[0], None, json.dumps(metadata)]) + failed_count += 1 + elif result[1] and len(result[1]) == 2: plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], metadata]) + results.append([result[0], point['the_geom'], json.dumps(metadata)]) success_count += 1 else: - results.append([result[0], None, metadata]) + results.append([result[0], None, json.dumps(metadata)]) empty_count = len(searches) - success_count - failed_count service_manager.quota_service.increment_success_service_use(success_count) diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index 240e6f7..6dd199c 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -18,6 +18,10 @@ SEARCH_FIXTURES = { 'wrong': [ StreetGeocoderSearch(id=100, address='deowpfjoepwjfopejwpofjewpojgf', city=None, state=None, country=None), + ], + 'error': [ + StreetGeocoderSearch(id=200, address=None, city=None, state=None, + country=None), ] } @@ -28,6 +32,9 @@ BULK_RESULTS_FIXTURES = { ], 'wrong': [ (100, [], {}) + ], + 'error': [ + (200, [], {'error': 'Something wrong happened'}) ] } @@ -38,6 +45,9 @@ EXPECTED_RESULTS_FIXTURES = { ], 'wrong': [ [100, None, '{}'] + ], + 'error': [ + [200, None, '{"error": "Something wrong happened"}'] ] } @@ -50,7 +60,8 @@ class TestRunStreetPointGeocoder(TestCase): self.plpy_mock.execute = MagicMock(return_value=[{'the_geom': point}]) self.logger_config_mock = MagicMock(min_log_level='debug', - log_file_path='/tmp/ptest.log') + log_file_path='/tmp/ptest.log', + rollbar_api_key=None) self.gd_mock = {'logger_config': self.logger_config_mock} self.geocoder_mock = Mock() @@ -144,4 +155,20 @@ class TestRunStreetPointGeocoder(TestCase): self.quota_service_mock.increment_empty_service_use. \ assert_called_once_with(len(SEARCH_FIXTURES['wrong'])) + def test_increment_mixed_error_service_use_on_complete_response(self): + searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['error'] + bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['error'] + self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) + result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, + self.geocoder_mock, + self.service_manager_mock, + 'any_username', None, searches) + + assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['error']) + + self.quota_service_mock.increment_success_service_use. \ + assert_called_once_with(len(SEARCH_FIXTURES['two'])) + self.quota_service_mock.increment_failed_service_use. \ + assert_called_once_with(len(SEARCH_FIXTURES['error'])) + From 1ff512839d60114cb645c061d9e259022045ee70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 13:01:46 +0200 Subject: [PATCH 147/265] Fixes empty results count --- .../cartodb_services/geocoder.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 8d5d456..f23695f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -24,7 +24,7 @@ def compose_address(street, city=None, state=None, country=None): return ', '.join(filter(None, [street, city, state, country])) -def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches): +def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches_string): plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] @@ -32,9 +32,15 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org success_count, failed_count = 0, 0 + try: + searches = json.loads(searches_string) + except Exception as e: + logger.error('Parsing searches', exception=e, data={'searches': searches_string}) + raise e + try: service_manager.assert_within_limits(quota=False) - geocode_results = geocoder.bulk_geocode(searches=searches) + geocode_results = geocoder.bulk_geocode(searches) results = [] if geocode_results: for result in geocode_results: @@ -94,20 +100,14 @@ class StreetPointBulkGeocoder: SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country'] - def bulk_geocode(self, searches): + def bulk_geocode(self, decoded_searches): """ - :param searches: array of StreetGeocoderSearch + :param decoded_searches: array of StreetGeocoderSearch :return: array of tuples with three elements: * id * latitude and longitude (array of two elements) * empty array (future use: metadata) """ - try: - decoded_searches = json.loads(searches) - except Exception as e: - self._logger.error('General error', exception=e) - raise e - street_geocoder_searches = [] for search in decoded_searches: search_id, address, city, state, country = \ From c5d9db61e64686ad24a6c7d877dd2d4b169748e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 15:48:32 +0200 Subject: [PATCH 148/265] Mapbox error handling --- .../cartodb_services/geocoder.py | 18 +++++++-- .../cartodb_services/mapbox/bulk_geocoder.py | 4 +- .../cartodb_services/mapbox/geocoder.py | 37 ++++++++++++------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index f23695f..cde5182 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -9,9 +9,6 @@ import json PRECISION_PRECISE = 'precise' PRECISION_INTERPOLATED = 'interpolated' -EMPTY_RESPONSE = [[], {}] - - def geocoder_metadata(relevance, precision, match_types): return { 'relevance': round(relevance, 2), @@ -20,6 +17,18 @@ def geocoder_metadata(relevance, precision, match_types): } +def geocoder_error_response(message): + return [[], {'error': message}] + + +# Single empty result +EMPTY_RESPONSE = [[], {}] +# HTTP 429 and related +TOO_MANY_REQUESTS_ERROR_RESPONSE = geocoder_error_response('Rate limit exceeded') +# Full empty _batch_geocode response +EMPTY_BATCH_RESPONSE = [] + + def compose_address(street, city=None, state=None, country=None): return ', '.join(filter(None, [street, city, state, country])) @@ -42,7 +51,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org service_manager.assert_within_limits(quota=False) geocode_results = geocoder.bulk_geocode(searches) results = [] - if geocode_results: + if not geocode_results == EMPTY_BATCH_RESPONSE: for result in geocode_results: if len(result) > 2: metadata = result[2] @@ -65,6 +74,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org results.append([result[0], None, json.dumps(metadata)]) empty_count = len(searches) - success_count - failed_count + logger.debug("--> Success: {}; empty: {}; failed: {}".format(success_count, empty_count, failed_count)) service_manager.quota_service.increment_success_service_use(success_count) service_manager.quota_service.increment_empty_service_use(empty_count) service_manager.quota_service.increment_failed_service_use(failed_count) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py index 2563d43..a939716 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py @@ -1,8 +1,6 @@ -import json, requests, time -from requests.adapters import HTTPAdapter +import requests from cartodb_services import StreetPointBulkGeocoder from cartodb_services.mapbox import MapboxGeocoder -from cartodb_services.tools.exceptions import ServiceException from iso3166 import countries from cartodb_services.tools.country import country_to_iso3 diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 2af2c6f..39d9da5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -5,7 +5,7 @@ Python client for the Mapbox Geocoder service. import json import requests from mapbox import Geocoder -from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE, EMPTY_BATCH_RESPONSE, TOO_MANY_REQUESTS_ERROR_RESPONSE, geocoder_error_response from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -69,7 +69,7 @@ class MapboxGeocoder(Traceable): result.append(EMPTY_RESPONSE) return result else: - return EMPTY_RESPONSE + return EMPTY_BATCH_RESPONSE def _extract_lng_lat_from_feature(self, feature): geometry = feature[ENTRY_GEOMETRY] @@ -118,9 +118,17 @@ class MapboxGeocoder(Traceable): :param city: :param state_province: :param country: Country ISO 3166 code - :return: [x, y] on success, [] on error + :return: [x, y] on success, raises ServiceException on error """ - return self.geocode_meta(searchtext, city, state_province, country)[0] + response = self.geocode_meta(searchtext, city, state_province, country) + if response: + error_message = response[1].get('error', None) + if error_message: + raise ServiceException(error_message, None) + else: + return response[0] + else: + return EMPTY_RESPONSE @qps_retry(qps=10) def geocode_meta(self, searchtext, city=None, state_province=None, @@ -138,7 +146,8 @@ class MapboxGeocoder(Traceable): free_search = ', '.join(address) - return self.geocode_free_text_meta([free_search], country)[0] + response = self.geocode_free_text_meta([free_search], country) + return response[0] if response else EMPTY_RESPONSE @qps_retry(qps=10) def geocode_free_text_meta(self, free_searches, country=None): @@ -157,24 +166,26 @@ class MapboxGeocoder(Traceable): if response.status_code == requests.codes.ok: return self._parse_geocoder_response(response.text) + elif response.status_code == requests.codes.too_many_requests: + return [TOO_MANY_REQUESTS_ERROR_RESPONSE] * len(free_searches) elif response.status_code == requests.codes.bad_request: - return EMPTY_RESPONSE + return EMPTY_BATCH_RESPONSE elif response.status_code == requests.codes.unprocessable_entity: - return EMPTY_RESPONSE + return EMPTY_BATCH_RESPONSE else: - raise ServiceException(response.status_code, response) + msg = "Unkown status: {}".format(response.status_code) + return [geocoder_error_response(msg)] * len(free_searches) except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down - self._logger.error('Timeout connecting to Mapbox geocoding server', - te) - raise ServiceException('Error geocoding {0} using Mapbox'.format( - free_search), None) + msg = 'Timeout connecting to Mapbox geocoding server' + self._logger.error(msg, te) + return [geocoder_error_response(msg)] * len(free_searches) except requests.ConnectionError as ce: # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to Mapbox geocoding server', exception=ce) - return EMPTY_RESPONSE + return EMPTY_BATCH_RESPONSE def _escape(self, free_search): # Semicolon is used to separate batch geocoding; there's no documented From bcb34d1ceacc8ed8f2dd038cfcc63e07fcdaeddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 15:57:32 +0200 Subject: [PATCH 149/265] Adjustments on street level fixtures --- test/integration/test_street_functions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 2d02bdb..a36cb18 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -11,7 +11,7 @@ class TestStreetFunctionsSetUp(TestCase): fixture_points = None GOOGLE_POINTS = { - 'Plaza Mayor, Valladolid': [-4.728252, 41.6517025], + 'Plaza Mayor 1, Valladolid': [-4.728252, 41.6517025], 'Paseo Zorrilla, Valladolid': [-4.7404453, 41.6314339], '1900 amphitheatre parkway': [-122.0875324, 37.4227968], '1901 amphitheatre parkway': [-122.0885504, 37.4238657], @@ -26,7 +26,7 @@ class TestStreetFunctionsSetUp(TestCase): } HERE_POINTS = { - 'Plaza Mayor, Valladolid': [-4.72979, 41.65258], + 'Plaza Mayor 1, Valladolid': [-4.72979, 41.65258], 'Paseo Zorrilla, Valladolid': [-4.73869, 41.63817], '1900 amphitheatre parkway': [-122.0879468, 37.4234763], '1901 amphitheatre parkway': [-122.0879253, 37.4238725], @@ -42,13 +42,13 @@ class TestStreetFunctionsSetUp(TestCase): TOMTOM_POINTS = HERE_POINTS.copy() TOMTOM_POINTS.update({ - 'Plaza Mayor, Valladolid': [-4.72183, 41.5826], + 'Plaza Mayor 1, Valladolid': [-4.7286, 41.6523], 'Paseo Zorrilla, Valladolid': [-4.74031, 41.63181], 'Valladolid': [-4.72838, 41.6542], 'Valladolid, Spain': [-4.72838, 41.6542], 'Madrid': [-3.70035, 40.42028], 'Logroño, Spain': [-2.44998, 42.46592], - 'Plaza España, Barcelona': [2.1497, 41.37516] + 'Plaza España, Barcelona': [2.14856, 41.37516] }) MAPBOX_POINTS = GOOGLE_POINTS.copy() @@ -174,7 +174,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ "'select 1 as cartodb_id, ''Spain'' as country, " \ "''Castilla y León'' as state, ''Valladolid'' as city, " \ - "''Plaza Mayor'' as street " \ + "''Plaza Mayor 1'' as street " \ "UNION " \ "select 2 as cartodb_id, ''Spain'' as country, " \ "''Castilla y León'' as state, ''Valladolid'' as city, " \ @@ -183,7 +183,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): response = self._run_authenticated(query) points_by_cartodb_id = { - 1: self.fixture_points['Plaza Mayor, Valladolid'], + 1: self.fixture_points['Plaza Mayor 1, Valladolid'], 2: self.fixture_points['Paseo Zorrilla, Valladolid'] } self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) From 07f5be92076070d1ace035d4984b61afba375097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 17:17:38 +0200 Subject: [PATCH 150/265] TomTom error handling --- .../cartodb_services/tomtom/bulk_geocoder.py | 13 ++++++++++-- .../cartodb_services/tomtom/geocoder.py | 20 ++++++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py index 8fc2e92..400cc2a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py @@ -1,6 +1,7 @@ import json, requests, time from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder +from cartodb_services.geocoder import geocoder_error_response from cartodb_services.tomtom import TomTomGeocoder from cartodb_services.tools.exceptions import ServiceException @@ -43,13 +44,21 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder): return results def _batch_geocode(self, searches): - location = self._send_batch(searches) - full_results = self._download_results(location) + full_results = self._geocode_searches(searches) results = [] for s, r in zip(searches, full_results): results.append((s[0], r[0], r[1])) return results + def _geocode_searches(self, searches): + try: + location = self._send_batch(searches) + return self._download_results(location) + except Exception as e: + msg = "Error running TomTom batch geocode: {}".format(e) + self._logger.error(msg, e) + return [geocoder_error_response(msg)] * len(searches) + def _send_batch(self, searches): body = {'batchItems': [{'query': self._query(s)} for s in searches]} request_params = { diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index 3f077bd..b5ec592 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -5,7 +5,7 @@ import json import requests from uritemplate import URITemplate from math import tanh -from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE, geocoder_error_response from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry @@ -73,7 +73,12 @@ class TomTomGeocoder(Traceable): @qps_retry(qps=5) def geocode(self, searchtext, city=None, state_province=None, country=None): - return self.geocode_meta(searchtext, city, state_province, country)[0] + response = self.geocode_meta(searchtext, city, state_province, country) + error_message = response[1].get('error', None) + if error_message: + raise ServiceException(error_message, None) + else: + return response[0] @qps_retry(qps=5) def geocode_meta(self, searchtext, city=None, state_province=None, @@ -106,10 +111,9 @@ class TomTomGeocoder(Traceable): except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down - self._logger.error('Timeout connecting to TomTom geocoding server', - te) - raise ServiceException('Error geocoding {0} using TomTom'.format( - searchtext), None) + msg = 'Timeout connecting to TomTom geocoding server' + self._logger.error(msg, te) + return geocoder_error_response(msg) except requests.ConnectionError as ce: # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to TomTom geocoding server', @@ -125,7 +129,9 @@ class TomTomGeocoder(Traceable): return EMPTY_RESPONSE else: msg = 'Unknown response {}: {}'.format(str(status_code), text) - raise ServiceException(msg, None) + self._logger.warning('Error parsing TomTom geocoding response', + data={'msg': msg}) + return geocoder_error_response(msg) def _parse_geocoder_response(self, response): json_response = json.loads(response) \ From 5d2303e1deb8d57d5bd2c3095182cba305cc479a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 17:23:08 +0200 Subject: [PATCH 151/265] Log a failed one if any --- .../python/cartodb_services/cartodb_services/geocoder.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index cde5182..c07ed08 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -51,6 +51,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org service_manager.assert_within_limits(quota=False) geocode_results = geocoder.bulk_geocode(searches) results = [] + a_failed_one = None if not geocode_results == EMPTY_BATCH_RESPONSE: for result in geocode_results: if len(result) > 2: @@ -64,6 +65,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org logger.warning('Geocoding error', data={"username": username, "orgname": orgname, "error": metadata['error']}) results.append([result[0], None, json.dumps(metadata)]) + a_failed_one = result failed_count += 1 elif result[1] and len(result[1]) == 2: plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) @@ -75,6 +77,13 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org empty_count = len(searches) - success_count - failed_count logger.debug("--> Success: {}; empty: {}; failed: {}".format(success_count, empty_count, failed_count)) + if a_failed_one: + logger.warning("failed geocoding", + data={ + "username": username, + "orgname": orgname, + "failed": str(a_failed_one) + }) service_manager.quota_service.increment_success_service_use(success_count) service_manager.quota_service.increment_empty_service_use(empty_count) service_manager.quota_service.increment_failed_service_use(failed_count) From faf9b7237b4936be14dec6435db88c92b9340c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 17:36:06 +0200 Subject: [PATCH 152/265] Adjustments on street level fixtures --- test/integration/test_street_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index a36cb18..b7557c9 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -26,7 +26,7 @@ class TestStreetFunctionsSetUp(TestCase): } HERE_POINTS = { - 'Plaza Mayor 1, Valladolid': [-4.72979, 41.65258], + 'Plaza Mayor 1, Valladolid': [-4.729, 41.65258], 'Paseo Zorrilla, Valladolid': [-4.73869, 41.63817], '1900 amphitheatre parkway': [-122.0879468, 37.4234763], '1901 amphitheatre parkway': [-122.0879253, 37.4238725], From 1b31c089ce09655a1f0530e654f44cdae009cc84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 18:16:40 +0200 Subject: [PATCH 153/265] Global error handling for batched geocoding --- .../cartodb_services/geocoder.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index c07ed08..289fde1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -121,7 +121,7 @@ class StreetPointBulkGeocoder: def bulk_geocode(self, decoded_searches): """ - :param decoded_searches: array of StreetGeocoderSearch + :param decoded_searches: JSON array :return: array of tuples with three elements: * id * latitude and longitude (array of two elements) @@ -136,10 +136,19 @@ class StreetPointBulkGeocoder: if len(street_geocoder_searches) > self.MAX_BATCH_SIZE: raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE)) - if self._should_use_batch(street_geocoder_searches): - return self._batch_geocode(street_geocoder_searches) - else: - return self._serial_geocode(street_geocoder_searches) + try: + if self._should_use_batch(street_geocoder_searches): + return self._batch_geocode(street_geocoder_searches) + else: + return self._serial_geocode(street_geocoder_searches) + except Exception as e: + msg = "Error running geocode: {}".format(e) + self._logger.error(msg, e) + errors = [geocoder_error_response(msg)] * len(decoded_searches) + results = [] + for s, r in zip(decoded_searches, errors): + results.append((s['id'], r[0], r[1])) + return results def _batch_geocode(self, street_geocoder_searches): raise NotImplementedError('Subclasses must implement _batch_geocode') From 8162bff2041da4eb4ba0fb267116f4ff01960030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 18:27:02 +0200 Subject: [PATCH 154/265] Serial geocoding error handling --- .../cartodb_services/here/bulk_geocoder.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 8316c44..772787c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -8,7 +8,7 @@ from collections import namedtuple from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder from cartodb_services.here import HereMapsGeocoder -from cartodb_services.geocoder import geocoder_metadata +from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException @@ -42,7 +42,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (search_id, address, city, state, country) = search - result = self.geocode_meta(searchtext=address, city=city, state=state, country=country) + try: + result = self.geocode_meta(searchtext=address, city=city, state=state, country=country) + except Exception as e: + self._logger.error("Error geocoding", e) + result = geocoder_error_response("Error geocoding") results.append((search_id, result[0], result[1])) return results From 4be3aa88fda2a555981a5b8e325854ca5d777f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 18:30:33 +0200 Subject: [PATCH 155/265] Constant extraction refactor --- .../cartodb_services/cartodb_services/here/geocoder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 23dd871..cbf2576 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -6,7 +6,7 @@ import requests from requests.adapters import HTTPAdapter from exceptions import * -from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata +from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE from cartodb_services.metrics import Traceable class HereMapsGeocoder(Traceable): @@ -90,7 +90,7 @@ class HereMapsGeocoder(Traceable): if value and value.strip(): params[key] = value if not params: - return [[], {}] + return EMPTY_RESPONSE return self._execute_geocode(params) def _execute_geocode(self, params): @@ -102,7 +102,7 @@ class HereMapsGeocoder(Traceable): return [self._extract_lng_lat_from_result(result), self._extract_metadata_from_result(result)] except IndexError: - return [[], {}] + return EMPTY_RESPONSE except KeyError as e: self._logger.error('params: {}'.format(params), e) raise MalformedResult() @@ -127,7 +127,7 @@ class HereMapsGeocoder(Traceable): self._logger.warning('Error 4xx trying to geocode street using HERE', data={"response": response.json(), "params": params}) - return [] + return EMPTY_RESPONSE else: self._logger.error('Error trying to geocode street using HERE', data={"response": response.json(), "params": From fc75f1afc8f326f49b0d15b0f6a9c961274d004d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 18:41:47 +0200 Subject: [PATCH 156/265] Google batch geocoder error handling --- .../cartodb_services/google/bulk_geocoder.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index 0e1ab1f..27a52b6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -1,7 +1,7 @@ from multiprocessing import Pool from exceptions import MalformedResult from cartodb_services import StreetPointBulkGeocoder -from cartodb_services.geocoder import compose_address +from cartodb_services.geocoder import compose_address, geocoder_error_response from cartodb_services.google import GoogleMapsGeocoder @@ -25,8 +25,13 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (cartodb_id, street, city, state, country) = search - lng_lat, metadata = self.geocode_meta(street, city, state, country) - results.append((cartodb_id, lng_lat, metadata)) + try: + lng_lat, metadata = self.geocode_meta(street, city, state, country) + result = (cartodb_id, lng_lat, metadata) + except Exception as e: + self._logger.error("Error geocoding", e) + result = geocoder_error_response("Error geocoding") + results.append(result) return results def _batch_geocode(self, searches): @@ -49,14 +54,12 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): try: lng_lat, metadata = self._process_results(bulk_result.get()) except Exception as e: - self._logger.error('Error at Google async_geocoder', e) - lng_lat, metadata = [[], {}] + msg = 'Error at Google async_geocoder' + self._logger.error(msg, e) + lng_lat, metadata = geocoder_error_response(msg) results.append((cartodb_id, lng_lat, metadata)) return results - except KeyError as e: - self._logger.error('KeyError error', exception=e) - raise MalformedResult() except Exception as e: self._logger.error('General error', exception=e) raise e From 3a5360c96cdb3929756921a9cde1819062bbaf78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 19:10:35 +0200 Subject: [PATCH 157/265] Refactor and fix for actual searches type --- .../test/test_cartodb_services_geocoder.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index 6dd199c..a58d5c2 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -1,6 +1,7 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- +import json from unittest import TestCase from mock import Mock, MagicMock from nose.tools import assert_not_equal, assert_equal, assert_true @@ -53,6 +54,17 @@ EXPECTED_RESULTS_FIXTURES = { class TestRunStreetPointGeocoder(TestCase): + def _run_geocoder(self, plpy=None, gd=None, geocoder=None, + service_manager=None, username=None, orgname=None, + searches=None): + return run_street_point_geocoder( + plpy if plpy else self.plpy_mock, + gd if gd else self.gd_mock, + geocoder if geocoder else self.geocoder_mock, + service_manager if service_manager else self.service_manager_mock, + username if username else 'any_username', + orgname if orgname else None, + json.dumps(searches) if searches else '[]') def setUp(self): point = [0,0] @@ -79,10 +91,8 @@ class TestRunStreetPointGeocoder(TestCase): searches = [] with(self.assertRaises(BaseException)): - run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + self._run_geocoder(service_manager=self.service_manager_mock, + searches=searches) self.quota_service_mock.increment_total_service_use. \ assert_called_once_with(len(searches)) @@ -96,10 +106,8 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] - result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = self._run_geocoder(service_manager=self.service_manager_mock, + searches=searches) assert_equal(result, []) self.quota_service_mock.increment_failed_service_use. \ @@ -116,10 +124,8 @@ class TestRunStreetPointGeocoder(TestCase): [2, [0, 0], '{}'], ] self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) - result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = self._run_geocoder(geocoder=self.geocoder_mock, + searches=searches) assert_equal(result, expected_results) self.quota_service_mock.increment_success_service_use. \ @@ -129,10 +135,8 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] results = [] self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) - result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = self._run_geocoder(geocoder=self.geocoder_mock, + searches=searches) assert_equal(result, results) @@ -143,10 +147,8 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['wrong'] bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['wrong'] self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) - result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = self._run_geocoder(geocoder=self.geocoder_mock, + searches=searches) assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['wrong']) @@ -159,10 +161,8 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['error'] bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['error'] self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) - result = run_street_point_geocoder(self.plpy_mock, self.gd_mock, - self.geocoder_mock, - self.service_manager_mock, - 'any_username', None, searches) + result = self._run_geocoder(geocoder=self.geocoder_mock, + searches=searches) assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['error']) From d060ab3d4117a582f6d58cb3a6feb2a450638bca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 19:11:23 +0200 Subject: [PATCH 158/265] Empty vs missing count detail --- .../cartodb_services/geocoder.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 289fde1..03ad014 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -39,7 +39,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org logger = Logger(logger_config) - success_count, failed_count = 0, 0 + success_count, failed_count, empty_count = 0, 0, 0 try: searches = json.loads(searches_string) @@ -54,16 +54,9 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org a_failed_one = None if not geocode_results == EMPTY_BATCH_RESPONSE: for result in geocode_results: - if len(result) > 2: - metadata = result[2] - else: - logger.warning('Geocoding without metadata', - data={"username": username, "orgname": orgname}) - metadata = {} + metadata = result[2] if len(result) > 2 else {} if metadata.get('error', None): - logger.warning('Geocoding error', - data={"username": username, "orgname": orgname, "error": metadata['error']}) results.append([result[0], None, json.dumps(metadata)]) a_failed_one = result failed_count += 1 @@ -74,18 +67,25 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org success_count += 1 else: results.append([result[0], None, json.dumps(metadata)]) + empty_count += 1 - empty_count = len(searches) - success_count - failed_count - logger.debug("--> Success: {}; empty: {}; failed: {}".format(success_count, empty_count, failed_count)) + missing_count = len(searches) - success_count - failed_count - empty_count + + logger.debug("--> Success: {}; empty: {}; missing: {}, failed: {}". + format(success_count, empty_count, missing_count, failed_count)) if a_failed_one: logger.warning("failed geocoding", data={ "username": username, "orgname": orgname, - "failed": str(a_failed_one) + "failed": str(a_failed_one), + "success_count": success_count, + "empty_count": empty_count, + "missing_count": missing_count, + "failed_count": failed_count }) service_manager.quota_service.increment_success_service_use(success_count) - service_manager.quota_service.increment_empty_service_use(empty_count) + service_manager.quota_service.increment_empty_service_use(empty_count + missing_count) service_manager.quota_service.increment_failed_service_use(failed_count) return results From fa3d7db5f89077849023275d3c1832a5916f59b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 21:52:54 +0200 Subject: [PATCH 159/265] Fix Google geocoder error handling --- .../cartodb_services/google/bulk_geocoder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py index 27a52b6..cde9f1c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py @@ -27,11 +27,10 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder): (cartodb_id, street, city, state, country) = search try: lng_lat, metadata = self.geocode_meta(street, city, state, country) - result = (cartodb_id, lng_lat, metadata) except Exception as e: self._logger.error("Error geocoding", e) - result = geocoder_error_response("Error geocoding") - results.append(result) + lng_lat, metadata = geocoder_error_response("Error geocoding") + results.append((cartodb_id, lng_lat, metadata)) return results def _batch_geocode(self, searches): From 80dcde2db09718e6caa05b75ca4ba4df35fc6c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 23 Jul 2018 22:00:16 +0200 Subject: [PATCH 160/265] Log Mapbox unknown status --- .../python/cartodb_services/cartodb_services/mapbox/geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py index 39d9da5..f50eff1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py @@ -163,7 +163,6 @@ class MapboxGeocoder(Traceable): response = self._geocoder.forward(address=free_search.decode('utf-8'), limit=1, country=country) - if response.status_code == requests.codes.ok: return self._parse_geocoder_response(response.text) elif response.status_code == requests.codes.too_many_requests: @@ -174,6 +173,7 @@ class MapboxGeocoder(Traceable): return EMPTY_BATCH_RESPONSE else: msg = "Unkown status: {}".format(response.status_code) + self._logger.warning(msg, data={"searches": free_searches}) return [geocoder_error_response(msg)] * len(free_searches) except requests.Timeout as te: # In case of timeout we want to stop the job because the server From 3524ee1e241afb2da7c21b0804b87683a722fbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 24 Jul 2018 11:31:05 +0200 Subject: [PATCH 161/265] Handle postprocessing error --- .../cartodb_services/geocoder.py | 28 ++++++---- .../test/test_cartodb_services_geocoder.py | 55 ++++++++++++++++--- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 03ad014..66dffc7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -55,19 +55,25 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org if not geocode_results == EMPTY_BATCH_RESPONSE: for result in geocode_results: metadata = result[2] if len(result) > 2 else {} - - if metadata.get('error', None): + try: + if metadata.get('error', None): + results.append([result[0], None, json.dumps(metadata)]) + a_failed_one = result + failed_count += 1 + elif result[1] and len(result[1]) == 2: + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) + point = plpy.execute(plan, result[1], 1)[0] + results.append([result[0], point['the_geom'], json.dumps(metadata)]) + success_count += 1 + else: + results.append([result[0], None, json.dumps(metadata)]) + empty_count += 1 + except Exception as e: + import sys + logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname}) + metadata['processing_error'] = 'Error: {}'.format(e.message) results.append([result[0], None, json.dumps(metadata)]) - a_failed_one = result failed_count += 1 - elif result[1] and len(result[1]) == 2: - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) - point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], json.dumps(metadata)]) - success_count += 1 - else: - results.append([result[0], None, json.dumps(metadata)]) - empty_count += 1 missing_count = len(searches) - success_count - failed_count - empty_count diff --git a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py index a58d5c2..ad36f72 100644 --- a/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py +++ b/server/lib/python/cartodb_services/test/test_cartodb_services_geocoder.py @@ -23,7 +23,15 @@ SEARCH_FIXTURES = { 'error': [ StreetGeocoderSearch(id=200, address=None, city=None, state=None, country=None), - ] + ], + 'broken_middle': [ + StreetGeocoderSearch(id=301, address='Paseo Zorrilla 1, Valladolid', + city=None, state=None, country=None), + StreetGeocoderSearch(id=302, address='Marsopolis', + city=None, state=None, country=None), + StreetGeocoderSearch(id=303, address='Paseo Zorrilla 2, Valladolid', + city=None, state=None, country=None) + ], } BULK_RESULTS_FIXTURES = { @@ -36,6 +44,11 @@ BULK_RESULTS_FIXTURES = { ], 'error': [ (200, [], {'error': 'Something wrong happened'}) + ], + 'broken_middle': [ + (301, [0, 0], {}), + (302, ['a', 'b'], {}), + (303, [0, 0], {}), ] } @@ -49,6 +62,11 @@ EXPECTED_RESULTS_FIXTURES = { ], 'error': [ [200, None, '{"error": "Something wrong happened"}'] + ], + 'broken_middle': [ + [301, [0, 0], '{}'], + [302, None, '{"processing_error": "Error: NO!"}'], + [303, [0, 0], '{}'], ] } @@ -87,7 +105,6 @@ class TestRunStreetPointGeocoder(TestCase): def test_count_increment_total_and_failed_service_use_on_error(self): self.service_manager_mock.assert_within_limits = \ Mock(side_effect=Exception('Fail!')) - searches = [] with(self.assertRaises(BaseException)): @@ -96,20 +113,17 @@ class TestRunStreetPointGeocoder(TestCase): self.quota_service_mock.increment_total_service_use. \ assert_called_once_with(len(searches)) - self.quota_service_mock.increment_failed_service_use. \ assert_called_once_with(len(searches)) def test_count_increment_failed_service_use_on_quota_error(self): self.service_manager_mock.assert_within_limits = \ Mock(side_effect=QuotaExceededException()) - searches = SEARCH_FIXTURES['two'] result = self._run_geocoder(service_manager=self.service_manager_mock, searches=searches) assert_equal(result, []) - self.quota_service_mock.increment_failed_service_use. \ assert_called_once_with(len(searches)) @@ -124,10 +138,10 @@ class TestRunStreetPointGeocoder(TestCase): [2, [0, 0], '{}'], ] self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) + result = self._run_geocoder(geocoder=self.geocoder_mock, searches=searches) assert_equal(result, expected_results) - self.quota_service_mock.increment_success_service_use. \ assert_called_once_with(len(results)) @@ -135,11 +149,11 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] results = [] self.geocoder_mock.bulk_geocode = MagicMock(return_value=results) + result = self._run_geocoder(geocoder=self.geocoder_mock, searches=searches) assert_equal(result, results) - self.quota_service_mock.increment_empty_service_use. \ assert_called_once_with(len(searches)) @@ -147,11 +161,11 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['wrong'] bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['wrong'] self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) + result = self._run_geocoder(geocoder=self.geocoder_mock, searches=searches) assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['wrong']) - self.quota_service_mock.increment_success_service_use. \ assert_called_once_with(len(SEARCH_FIXTURES['two'])) self.quota_service_mock.increment_empty_service_use. \ @@ -161,14 +175,37 @@ class TestRunStreetPointGeocoder(TestCase): searches = SEARCH_FIXTURES['two'] + SEARCH_FIXTURES['error'] bulk_results = BULK_RESULTS_FIXTURES['two'] + BULK_RESULTS_FIXTURES['error'] self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) + result = self._run_geocoder(geocoder=self.geocoder_mock, searches=searches) assert_equal(result, EXPECTED_RESULTS_FIXTURES['two'] + EXPECTED_RESULTS_FIXTURES['error']) - self.quota_service_mock.increment_success_service_use. \ assert_called_once_with(len(SEARCH_FIXTURES['two'])) self.quota_service_mock.increment_failed_service_use. \ assert_called_once_with(len(SEARCH_FIXTURES['error'])) + def test_controlled_failure_on_query_break(self): + searches = SEARCH_FIXTURES['broken_middle'] + bulk_results = BULK_RESULTS_FIXTURES['broken_middle'] + self.geocoder_mock.bulk_geocode = MagicMock(return_value=bulk_results) + def break_on_302(*args): + if len(args) == 3: + plan, values, limit = args + if values[0] == 'a': + raise Exception('NO!') + + return [{'the_geom': [0,0]}] + self.plpy_mock.execute = break_on_302 + + result = self._run_geocoder(geocoder=self.geocoder_mock, + searches=searches) + + assert_equal(result, EXPECTED_RESULTS_FIXTURES['broken_middle']) + self.quota_service_mock.increment_success_service_use. \ + assert_called_once_with(2) + self.quota_service_mock.increment_failed_service_use. \ + assert_called_once_with(1) + + From c6720bf689cee021dffe418b222396cfbf2798e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 24 Jul 2018 11:57:54 +0200 Subject: [PATCH 162/265] Better debug message --- .../cartodb_services/cartodb_services/geocoder.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index 66dffc7..a2c250c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -77,8 +77,6 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org missing_count = len(searches) - success_count - failed_count - empty_count - logger.debug("--> Success: {}; empty: {}; missing: {}, failed: {}". - format(success_count, empty_count, missing_count, failed_count)) if a_failed_one: logger.warning("failed geocoding", data={ @@ -90,6 +88,16 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org "missing_count": missing_count, "failed_count": failed_count }) + else: + logger.debug("finished geocoding", + data={ + "username": username, + "orgname": orgname, + "success_count": success_count, + "empty_count": empty_count, + "missing_count": missing_count, + "failed_count": failed_count + }) service_manager.quota_service.increment_success_service_use(success_count) service_manager.quota_service.increment_empty_service_use(empty_count + missing_count) service_manager.quota_service.increment_failed_service_use(failed_count) From 11ec6075c3fae060017086f0b37023b6f0a022d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 24 Jul 2018 11:59:29 +0200 Subject: [PATCH 163/265] Python library version 0.19.1 --- NEWS.md | 2 +- server/lib/python/cartodb_services/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9883417..4571e01 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ Jul 19th, 2018 ============== -* Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.0` of the Python library. +* Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.1` of the Python library. * Support for batch street-level geocoding. May 7th, 2018 diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 3ea820e..7699f06 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.19.0', + version='0.19.1', description='CartoDB Services API Python Library', From 075f602a7f3ed5bd5fa76407c8d89afffca65a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 25 Jul 2018 11:22:31 +0200 Subject: [PATCH 164/265] WELL_KNOWN_SHAPE and WELL_KNOWN_LENGTH fixture update --- .../test/test_mapboxrouting.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_mapboxrouting.py b/server/lib/python/cartodb_services/test/test_mapboxrouting.py index 160b2a5..2101535 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapboxrouting.py @@ -17,17 +17,16 @@ VALID_PROFILE = DEFAULT_PROFILE INVALID_PROFILE = 'invalid_profile' WELL_KNOWN_SHAPE = [(40.73312, -73.98891), (40.73353, -73.98987), - (40.73398, -73.99095), (40.73453, -73.99227), - (40.73531, -73.99412), (40.73467, -73.99459), - (40.73442, -73.99477), (40.73435, -73.99482), - (40.73403, -73.99505), (40.73344, -73.99549), - (40.73286, -73.9959), (40.73226, -73.99635), - (40.73186, -73.99664), (40.73147, -73.99693), - (40.73141, -73.99698), (40.73147, -73.99707), - (40.73219, -73.99856), (40.73222, -73.99861), - (40.73225, -73.99868), (40.73293, -74.00007), - (40.733, -74.00001)] -WELL_KNOWN_LENGTH = 1317.9 + (40.73398, -73.99095), (40.73321, -73.99111), + (40.73245, -73.99129), (40.7333, -73.99332), + (40.7338, -73.99449), (40.73403, -73.99505), + (40.73344, -73.99549), (40.73286, -73.9959), + (40.73226, -73.99635), (40.73186, -73.99664), + (40.73147, -73.99693), (40.73141, -73.99698), + (40.73147, -73.99707), (40.73219, -73.99856), + (40.73222, -73.99861), (40.73225, -73.99868), + (40.73293, -74.00007), (40.733, -74.00001)] +WELL_KNOWN_LENGTH = 1384.8 class MapboxRoutingTestCase(unittest.TestCase): From a32e90ea8ad93ea7c0fd3b761c985668f5571be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 2 Aug 2018 12:59:45 +0200 Subject: [PATCH 165/265] carto-package.json proposal --- client/carto-package.json | 17 +++++++++++ server/extension/carto-package.json | 30 +++++++++++++++++++ .../cartodb_services/carto-package.json | 11 +++++++ 3 files changed, 58 insertions(+) create mode 100644 client/carto-package.json create mode 100644 server/extension/carto-package.json create mode 100644 server/lib/python/cartodb_services/carto-package.json diff --git a/client/carto-package.json b/client/carto-package.json new file mode 100644 index 0000000..8e9608d --- /dev/null +++ b/client/carto-package.json @@ -0,0 +1,17 @@ +{ + "name": "dataservices-api-client-extension", + "versions": { + "0.25.0": { + "tag": "0.25.0-client", + "works_with": { + "dataservices-api-server-extension": "^0.32.0" + }, + "requires": { + "postgresql": "^10.0.0", + "postgis": "^2.4", + "cartodb-postgresql": "^0.23.0" + } + } + } +} + diff --git a/server/extension/carto-package.json b/server/extension/carto-package.json new file mode 100644 index 0000000..fd0d32a --- /dev/null +++ b/server/extension/carto-package.json @@ -0,0 +1,30 @@ +{ + "name": "dataservices-api-server-extension", + "versions": { + "0.32.0": { + "tag": "0.32.0-server", + "works_with": { + "dataservices-api-server-python-lib": "^0.19.0" + }, + "requires": { + "postgresql": "^10.0.0", + "postgis": "^2.4", + "cartodb-postgresql": "^0.23.0" + } + }, + "0.32.0-customer": { + "tag": "0.32.0-server-customer", + "works_with": { + "dataservices-api-server-python-lib": { + "git-repository": "https://github.com/CUSTOMER/dataservices-api", + "manifest": "/server/extension/carto-package.json", + "version": "0.19.0-experimental" + } + }, + "requires": { + "postgresql": "^10.0.0", + "postgis": "^2.4", + "cartodb-postgresql": "^0.23.0" + } + } +} diff --git a/server/lib/python/cartodb_services/carto-package.json b/server/lib/python/cartodb_services/carto-package.json new file mode 100644 index 0000000..9b9e655 --- /dev/null +++ b/server/lib/python/cartodb_services/carto-package.json @@ -0,0 +1,11 @@ +{ + "name": "dataservices-api-server-python-lib", + "versions": { + "0.19.0": { + "tag": "python-0.19.0", + "requires": { + "python": "2.7" + } + } + } +} From c1c671755c48e7a5823d8e6ee47a4c6cc4ac5d5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 13 Aug 2018 16:20:59 +0200 Subject: [PATCH 166/265] Final carto-package.json format --- client/carto-package.json | 20 +++++------- server/extension/carto-package.json | 32 +++++-------------- .../cartodb_services/carto-package.json | 9 ++---- 3 files changed, 19 insertions(+), 42 deletions(-) diff --git a/client/carto-package.json b/client/carto-package.json index 8e9608d..1ec7a40 100644 --- a/client/carto-package.json +++ b/client/carto-package.json @@ -1,17 +1,13 @@ { "name": "dataservices-api-client-extension", - "versions": { - "0.25.0": { - "tag": "0.25.0-client", - "works_with": { - "dataservices-api-server-extension": "^0.32.0" - }, - "requires": { - "postgresql": "^10.0.0", - "postgis": "^2.4", - "cartodb-postgresql": "^0.23.0" - } + "current_version": { + "requires": { + "postgresql": "^10.0.0", + "postgis": "^2.4", + "cartodb-extension": "^0.23.0" + }, + "works_with": { + "dataservices-api-server-extension": "^0.32.0" } } } - diff --git a/server/extension/carto-package.json b/server/extension/carto-package.json index fd0d32a..29514f7 100644 --- a/server/extension/carto-package.json +++ b/server/extension/carto-package.json @@ -1,30 +1,14 @@ { "name": "dataservices-api-server-extension", - "versions": { - "0.32.0": { - "tag": "0.32.0-server", - "works_with": { - "dataservices-api-server-python-lib": "^0.19.0" - }, - "requires": { - "postgresql": "^10.0.0", - "postgis": "^2.4", - "cartodb-postgresql": "^0.23.0" - } + "current_version": { + "requires": { + "postgresql": "^10.0.0", + "postgis": "^2.4", + "cartodb-postgresql": "^0.23.0" }, - "0.32.0-customer": { - "tag": "0.32.0-server-customer", - "works_with": { - "dataservices-api-server-python-lib": { - "git-repository": "https://github.com/CUSTOMER/dataservices-api", - "manifest": "/server/extension/carto-package.json", - "version": "0.19.0-experimental" - } - }, - "requires": { - "postgresql": "^10.0.0", - "postgis": "^2.4", - "cartodb-postgresql": "^0.23.0" + "works_with": { + "dataservices-api-server-python-lib": "^0.19.1", + "observatory-server-extension": "^1.9.0" } } } diff --git a/server/lib/python/cartodb_services/carto-package.json b/server/lib/python/cartodb_services/carto-package.json index 9b9e655..47a1277 100644 --- a/server/lib/python/cartodb_services/carto-package.json +++ b/server/lib/python/cartodb_services/carto-package.json @@ -1,11 +1,8 @@ { "name": "dataservices-api-server-python-lib", - "versions": { - "0.19.0": { - "tag": "python-0.19.0", - "requires": { - "python": "2.7" - } + "current_version": { + "requires": { + "python": "2.7" } } } From ed2e87f4ca2fac07879172eef2be08bfdd6e58e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 14 Aug 2018 09:27:59 +0200 Subject: [PATCH 167/265] Fix test fixture --- .../test/test_mapboxrouting.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_mapboxrouting.py b/server/lib/python/cartodb_services/test/test_mapboxrouting.py index 2101535..160b2a5 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapboxrouting.py @@ -17,16 +17,17 @@ VALID_PROFILE = DEFAULT_PROFILE INVALID_PROFILE = 'invalid_profile' WELL_KNOWN_SHAPE = [(40.73312, -73.98891), (40.73353, -73.98987), - (40.73398, -73.99095), (40.73321, -73.99111), - (40.73245, -73.99129), (40.7333, -73.99332), - (40.7338, -73.99449), (40.73403, -73.99505), - (40.73344, -73.99549), (40.73286, -73.9959), - (40.73226, -73.99635), (40.73186, -73.99664), - (40.73147, -73.99693), (40.73141, -73.99698), - (40.73147, -73.99707), (40.73219, -73.99856), - (40.73222, -73.99861), (40.73225, -73.99868), - (40.73293, -74.00007), (40.733, -74.00001)] -WELL_KNOWN_LENGTH = 1384.8 + (40.73398, -73.99095), (40.73453, -73.99227), + (40.73531, -73.99412), (40.73467, -73.99459), + (40.73442, -73.99477), (40.73435, -73.99482), + (40.73403, -73.99505), (40.73344, -73.99549), + (40.73286, -73.9959), (40.73226, -73.99635), + (40.73186, -73.99664), (40.73147, -73.99693), + (40.73141, -73.99698), (40.73147, -73.99707), + (40.73219, -73.99856), (40.73222, -73.99861), + (40.73225, -73.99868), (40.73293, -74.00007), + (40.733, -74.00001)] +WELL_KNOWN_LENGTH = 1317.9 class MapboxRoutingTestCase(unittest.TestCase): From d9f647504a014b5b320eaa9a6d52afa852cb1cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Tue, 14 Aug 2018 09:39:42 +0200 Subject: [PATCH 168/265] Disable Mapbox Matrix API usage at tests --- .../python/cartodb_services/test/metrics/test_config.py | 7 ++++--- .../lib/python/cartodb_services/test/test_mapboxisoline.py | 4 +--- .../lib/python/cartodb_services/test/test_mapboxmatrix.py | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 9ab9f34..4a14a8a 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -173,7 +173,8 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] + # Don't test mapbox. See CartoDB/cartodb-management/issues/5199" + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() @@ -233,8 +234,8 @@ class TestIsolinesUserConfig(TestCase): class TestIsolinesOrgConfig(TestCase): - - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] + # Don't test mapbox. See CartoDB/cartodb-management/issues/5199" + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() diff --git a/server/lib/python/cartodb_services/test/test_mapboxisoline.py b/server/lib/python/cartodb_services/test/test_mapboxisoline.py index ce80d0e..091ac7c 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxisoline.py +++ b/server/lib/python/cartodb_services/test/test_mapboxisoline.py @@ -3,15 +3,13 @@ from mock import Mock from cartodb_services.mapbox.isolines import MapboxIsolines from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE from cartodb_services.mapbox.matrix_client import MapboxMatrixClient -from cartodb_services.mapbox.routing import MapboxRouting from cartodb_services.tools import Coordinate -from cartodb_services.tools.coordinates import (validate_coordinates, - marshall_coordinates) from credentials import mapbox_api_key VALID_ORIGIN = Coordinate(-73.989, 40.733) +@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199") class MapboxIsolinesTestCase(unittest.TestCase): def setUp(self): diff --git a/server/lib/python/cartodb_services/test/test_mapboxmatrix.py b/server/lib/python/cartodb_services/test/test_mapboxmatrix.py index 579810a..b1ce42e 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxmatrix.py +++ b/server/lib/python/cartodb_services/test/test_mapboxmatrix.py @@ -20,6 +20,7 @@ VALID_PROFILE = DEFAULT_PROFILE INVALID_PROFILE = 'invalid_profile' +@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199") class MapboxMatrixTestCase(unittest.TestCase): def setUp(self): self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(), From bd391f4bf421aa0d9a5e8353393289759bc49773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guirao=20Rodr=C3=ADguez?= Date: Fri, 17 Aug 2018 10:54:24 +0200 Subject: [PATCH 169/265] Component change and versioning using more numbers --- server/extension/carto-package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/extension/carto-package.json b/server/extension/carto-package.json index 29514f7..7e9187c 100644 --- a/server/extension/carto-package.json +++ b/server/extension/carto-package.json @@ -3,8 +3,8 @@ "current_version": { "requires": { "postgresql": "^10.0.0", - "postgis": "^2.4", - "cartodb-postgresql": "^0.23.0" + "postgis": "^2.4.0.0", + "carto_postgresql_ext": "^0.23.0" }, "works_with": { "dataservices-api-server-python-lib": "^0.19.1", From 61aaa0804f16cdcdf41892dfd8fc2ad0f64d4267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guirao=20Rodr=C3=ADguez?= Date: Fri, 17 Aug 2018 10:56:07 +0200 Subject: [PATCH 170/265] Changes in component names and versions --- client/carto-package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/carto-package.json b/client/carto-package.json index 1ec7a40..30d0750 100644 --- a/client/carto-package.json +++ b/client/carto-package.json @@ -3,8 +3,8 @@ "current_version": { "requires": { "postgresql": "^10.0.0", - "postgis": "^2.4", - "cartodb-extension": "^0.23.0" + "postgis": "^2.4.0.0", + "carto_sql_ext": "^0.23.0" }, "works_with": { "dataservices-api-server-extension": "^0.32.0" From 9c8647ebd4c5ddbb034272f660bb7ef746362402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guirao=20Rodr=C3=ADguez?= Date: Fri, 17 Aug 2018 10:57:52 +0200 Subject: [PATCH 171/265] Changes in component names and versions --- server/lib/python/cartodb_services/carto-package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/carto-package.json b/server/lib/python/cartodb_services/carto-package.json index 47a1277..1f0f1c8 100644 --- a/server/lib/python/cartodb_services/carto-package.json +++ b/server/lib/python/cartodb_services/carto-package.json @@ -2,7 +2,7 @@ "name": "dataservices-api-server-python-lib", "current_version": { "requires": { - "python": "2.7" + "python": "~2.7.0" } } } From a563abb7abf5450478f4538ed7023e46c7860c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guirao=20Rodr=C3=ADguez?= Date: Fri, 17 Aug 2018 10:58:56 +0200 Subject: [PATCH 172/265] Update carto-package.json --- client/carto-package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/carto-package.json b/client/carto-package.json index 30d0750..df769d7 100644 --- a/client/carto-package.json +++ b/client/carto-package.json @@ -4,7 +4,7 @@ "requires": { "postgresql": "^10.0.0", "postgis": "^2.4.0.0", - "carto_sql_ext": "^0.23.0" + "carto_postgresql_ext": "^0.23.0" }, "works_with": { "dataservices-api-server-extension": "^0.32.0" From 44da876b4cd3522063ddd84bbe7eecd4f80eb64f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:15:29 +0200 Subject: [PATCH 173/265] Version 0.33.0 initial commit --- ...db_dataservices_server--0.32.0--0.33.0.sql | 5 + ...db_dataservices_server--0.33.0--0.32.0.sql | 5 + ...ql => cdb_dataservices_server--0.33.0.sql} | 0 .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.31.0--0.32.0.sql | 0 ...db_dataservices_server--0.32.0--0.31.0.sql | 0 .../cdb_dataservices_server--0.32.0.sql | 3801 +++++++++++++++++ 7 files changed, 3812 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql rename server/extension/{cdb_dataservices_server--0.32.0.sql => cdb_dataservices_server--0.33.0.sql} (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.31.0--0.32.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.32.0--0.31.0.sql (100%) create mode 100644 server/extension/old_versions/cdb_dataservices_server--0.32.0.sql diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql new file mode 100644 index 0000000..0494d33 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.33.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql new file mode 100644 index 0000000..7da98ea --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.32.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.33.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.32.0.sql rename to server/extension/cdb_dataservices_server--0.33.0.sql diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 908a91d..fb4506e 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.32.0' +default_version = '0.33.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.31.0--0.32.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.31.0--0.32.0.sql diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.32.0--0.31.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.32.0--0.31.0.sql diff --git a/server/extension/old_versions/cdb_dataservices_server--0.32.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.32.0.sql new file mode 100644 index 0000000..42d9b9e --- /dev/null +++ b/server/extension/old_versions/cdb_dataservices_server--0.32.0.sql @@ -0,0 +1,3801 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; From fba933cc88ae09a374c17d0890fe17f8c1b006c7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:18:10 +0200 Subject: [PATCH 174/265] Remove obs_snapshot_config --- .../cdb_dataservices_server--0.32.0--0.33.0.sql | 3 ++- .../cdb_dataservices_server--0.33.0--0.32.0.sql | 15 ++++++++++++++- server/extension/sql/15_config_helper.sql | 14 -------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql index 0494d33..ff9f166 100644 --- a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql @@ -2,4 +2,5 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.33.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config; diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql index 7da98ea..1e883ec 100644 --- a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql @@ -2,4 +2,17 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.32.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 31d3331..6e4d884 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -90,20 +90,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_obs_snapshot_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import ObservatorySnapshotConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = obs_snapshot_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_obs_config_{0}".format(username) From 0e95c5ff90649ad7199f105458675a1b555f4783 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:25:12 +0200 Subject: [PATCH 175/265] Now the snapshot functions use the obs_genral config --- ...db_dataservices_server--0.32.0--0.33.0.sql | 198 ++++++++++++++++++ ...db_dataservices_server--0.33.0--0.32.0.sql | 198 ++++++++++++++++++ .../sql/110_data_observatory_augmentation.sql | 20 +- 3 files changed, 406 insertions(+), 10 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql index ff9f166..610916d 100644 --- a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql @@ -4,3 +4,201 @@ -- HERE goes your code to upgrade/downgrade DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql index 1e883ec..1a30b9d 100644 --- a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql @@ -16,3 +16,201 @@ RETURNS boolean AS $$ GD[cache_key] = obs_snapshot_config return True $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index b5cde1c..95f8f20 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -10,8 +10,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( RETURNS text AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] return user_obs_config.connection_str $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -41,8 +41,8 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] @@ -94,8 +94,8 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] @@ -150,8 +150,8 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] @@ -201,8 +201,8 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] From 6cc3cda6e0167ef0d50a8900068b21655c7d90ab Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:28:55 +0200 Subject: [PATCH 176/265] Remove obs_snapshot_config from python library --- .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 25 ------------- .../test/metrics/test_config.py | 36 +------------------ .../cartodb_services/test/test_helper.py | 3 -- .../test/test_quota_service.py | 33 ----------------- 5 files changed, 2 insertions(+), 97 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index b0a4181..edca0ee 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,4 +1,4 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig from quota import QuotaService from user import UserMetricsService from log import metrics, MetricsDataGatherer, Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index f0594bc..3e45694 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -86,28 +86,6 @@ class DataObservatoryConfig(ServiceConfig): return 'data observatory' -class ObservatorySnapshotConfig(DataObservatoryConfig): - - SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit' - QUOTA_KEY = 'obs_snapshot_quota' - PERIOD_END_DATE = 'period_end_date' - - def __init__(self, redis_connection, db_conn, username, orgname=None): - super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn, - username, orgname) - self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) - if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': - self._soft_limit = True - else: - self._soft_limit = False - self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY) - self._connection_str = self._db_config.data_observatory_connection_str - - @property - def service_type(self): - return 'obs_snapshot' - - class ObservatoryConfig(DataObservatoryConfig): SOFT_LIMIT_KEY = 'soft_obs_general_limit' @@ -890,7 +868,6 @@ class ServicesRedisConfig: QUOTA_KEY = 'geocoding_quota' ISOLINES_QUOTA_KEY = 'here_isolines_quota' ROUTING_QUOTA_KEY = 'mapzen_routing_quota' - OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota' OBS_GENERAL_QUOTA_KEY = 'obs_general_quota' PERIOD_END_DATE = 'period_end_date' GEOCODER_PROVIDER_KEY = 'geocoder_provider' @@ -934,8 +911,6 @@ class ServicesRedisConfig: user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] if self.ROUTING_QUOTA_KEY in org_config: user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY] - if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: - user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] if self.OBS_GENERAL_QUOTA_KEY in org_config: user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY] if self.PERIOD_END_DATE in org_config: diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 4a14a8a..bf7d6b5 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -344,41 +344,7 @@ class TestDataObservatoryUserConfig(TestCase): self.redis_conn = MockRedis() plpy_mock_config() - def test_should_return_config_for_obs_snapshot(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', - quota=100, end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.monthly_quota == 100 - assert do_config.soft_limit is False - assert do_config.period_end_date.date() == yesterday.date() - - def test_should_return_true_if_soft_limit_is_true_in_redis(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', - quota=0, soft_limit=True, end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.soft_limit is True - - def test_should_return_0_if_quota_is_0_in_redis(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', - quota=0, end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.monthly_quota == 0 - - def test_should_return_0_if_quota_is_empty_in_redis(self): - yesterday = datetime.today() - timedelta(days=1) - build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', - quota='', end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - 'test_user') - assert do_config.monthly_quota == 0 - - def test_should_return_config_for_obs_snapshot(self): + def test_should_return_config_for_obs_config(self): yesterday = datetime.today() - timedelta(days=1) build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory', quota=100, end_date=yesterday) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index d8bccea..dcdbbf5 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -28,9 +28,7 @@ def build_redis_user_config(redis_conn, username, service, quota=100, redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota)) redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower()) elif service is 'data_observatory': - redis_conn.hset(user_redis_name, 'obs_snapshot_quota', str(quota)) redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota)) - redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', str(soft_limit).lower()) redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower()) redis_conn.hset(user_redis_name, 'google_maps_client_id', '') @@ -57,7 +55,6 @@ def build_redis_org_config(redis_conn, orgname, service, quota=100, redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota)) elif service is 'data_observatory': if quota is not None: - redis_conn.hset(org_redis_name, 'obs_snapshot_quota', str(quota)) redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota)) redis_conn.hset(org_redis_name, 'google_maps_client_id', '') diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index a3aa4f4..ce1ca21 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -181,28 +181,6 @@ class TestQuotaService(TestCase): # Quick workaround so we don't take into account numer of credits # spent for users that have defined the quota. # See https://github.com/CartoDB/bigmetadata/issues/215 - def test_should_check_user_obs_snapshot_quota_correctly(self): - qs = self.__build_obs_snapshot_quota_service('test_user') - qs.increment_success_service_use() - assert qs.check_user_quota() is True - qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is True - - def test_should_check_org_obs_snapshot_quota_correctly(self): - qs = self.__build_obs_snapshot_quota_service('test_user', - orgname='testorg') - qs.increment_success_service_use() - assert qs.check_user_quota() is True - qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is True - - def test_should_check_user_obs_quota_correctly(self): - qs = self.__build_obs_snapshot_quota_service('test_user') - qs.increment_success_service_use() - assert qs.check_user_quota() is True - qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is True - def test_should_check_org_obs_quota_correctly(self): qs = self.__build_obs_quota_service('test_user', orgname='testorg') @@ -249,17 +227,6 @@ class TestQuotaService(TestCase): username, orgname) return QuotaService(isolines_config, redis_connection=self.redis_conn) - def __build_obs_snapshot_quota_service(self, username, quota=100, - provider='obs_snapshot', - orgname=None, - soft_limit=False, - end_date=datetime.today()): - self.__prepare_quota_service(username, 'data_observatory', quota, - None, orgname, soft_limit, end_date) - do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, - username, orgname) - return QuotaService(do_config, redis_connection=self.redis_conn) - def __build_obs_quota_service(self, username, quota=100, provider='obs_general', orgname=None, From a8e96366a55684c14835a3ef183708fa7177bd67 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:40:22 +0200 Subject: [PATCH 177/265] Bump for the python library version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 7699f06..ea9ab90 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.19.1', + version='0.20.0', description='CartoDB Services API Python Library', From 073c527a62da3298be2e4167b9ba61370acf789c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 21 Aug 2018 18:42:16 +0200 Subject: [PATCH 178/265] Update NEWS.md --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index 4571e01..e1bb069 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ + +Aug ?th, 2018 +============== +* Version `0.33.0` of the server, and `0.20.0` of the Python library. + * Remove the obs_snapshot quota and now the snapshot functions uses obs_general quota + Jul 19th, 2018 ============== * Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.1` of the Python library. From 9398ec05242b5eacbf4327b03b7b6cc9e991ac30 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 27 Aug 2018 11:56:15 +0200 Subject: [PATCH 179/265] Updated NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e1bb069..9de5aaa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,5 @@ -Aug ?th, 2018 +Aug 27th, 2018 ============== * Version `0.33.0` of the server, and `0.20.0` of the Python library. * Remove the obs_snapshot quota and now the snapshot functions uses obs_general quota From 652242a8f88d968721d74a79fe2eedfc55e8f94c Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Wed, 5 Sep 2018 17:02:02 +0200 Subject: [PATCH 180/265] Permissions check and client templates --- client/renderer/interface.yaml | 84 +++++++++++++++++++ client/renderer/sql-template-renderer | 12 +++ .../templates/20_public_functions.erb | 10 ++- 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 54805cd..97ad5cc 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -1,33 +1,51 @@ --- - name: cdb_geocode_admin0_polygon return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: country_name, type: text } - name: cdb_geocode_admin1_polygon return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: admin1_name, type: text } - name: cdb_geocode_admin1_polygon return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: admin1_name, type: text } - { name: country_name, type: text } - name: cdb_geocode_namedplace_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: city_name, type: text} - name: cdb_geocode_namedplace_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: city_name, type: text} - { name: country_name, type: text} - name: cdb_geocode_namedplace_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: city_name, type: text} - { name: admin1_name, type: text} @@ -35,35 +53,53 @@ - name: cdb_geocode_postalcode_polygon return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: postal_code, type: text} - { name: country_name, type: text} - name: cdb_geocode_postalcode_polygon return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: postal_code, type: double precision} - { name: country_name, type: text} - name: cdb_geocode_postalcode_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: postal_code, type: text} - { name: country_name, type: text} - name: cdb_geocode_postalcode_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: postal_code, type: double precision} - { name: country_name, type: text} - name: cdb_geocode_ipaddress_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: ip_address, type: text} - name: cdb_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -74,11 +110,17 @@ return_type: SETOF cdb_dataservices_client.geocoding multi_row: true multi_field: true + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searches, type: jsonb } # Array of JSON objects with id, address, city, state and country fields - name: cdb_here_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -87,6 +129,9 @@ - name: cdb_google_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -95,6 +140,9 @@ - name: cdb_mapbox_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -103,6 +151,9 @@ - name: cdb_tomtom_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -111,6 +162,9 @@ - name: cdb_mapzen_geocode_street_point return_type: Geometry + requires_permission: true + permission_name: geocoding + permission_error: Geocoding is not allowed params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -121,6 +175,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -131,6 +188,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -141,6 +201,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -151,6 +214,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -161,6 +227,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -171,6 +240,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -181,6 +253,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -191,6 +266,9 @@ return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines are not allowed params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -200,6 +278,9 @@ - name: cdb_route_point_to_point return_type: cdb_dataservices_client.simple_route multi_field: true + requires_permission: true + permission_name: routing + permission_error: Routing is not allowed params: - { name: origin, type: "geometry(Point, 4326)" } - { name: destination, type: "geometry(Point, 4326)" } @@ -210,6 +291,9 @@ - name: cdb_route_with_waypoints return_type: cdb_dataservices_client.simple_route multi_field: true + requires_permission: true + permission_name: routing + permission_error: Routing is not allowed params: - { name: waypoints, type: "geometry(Point, 4326)[]" } - { name: mode, type: text } diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index 5741030..23bd1d1 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -28,6 +28,18 @@ class SqlTemplateRenderer @function_signature['return_type'] end + def requires_permission + @function_signature['requires_permission'] + end + + def permission_name + @function_signature['permission_name'] + end + + def permission_error + @function_signature['permission_error'] + end + def multi_field @function_signature['multi_field'] end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 5ce9311..9b32b59 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -13,7 +13,15 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %> + <% unless superuser_function? -%> + <% if requires_permission %> + SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN + RAISE EXCEPTION '<%= permission_error %>'; + END IF; + <% else %> + SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text); + <% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From 2a2ae481fb32662724ebb8278f9185e951739095 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Wed, 5 Sep 2018 17:28:15 +0200 Subject: [PATCH 181/265] Permissions check and client templates --- client/renderer/templates/20_public_functions.erb | 6 +++++- client/sql/15_config_management.sql | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 9b32b59..f9696db 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -9,13 +9,17 @@ RETURNS <%= return_type %> AS $$ DECLARE <%= return_declaration if not multi_row %> <%= user_org_declaration %> + <% unless superuser_function? -%> + appname TEXT; + apikey_permissions JSON; + <% end %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; <% unless superuser_function? -%> - <% if requires_permission %> SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + <% if requires_permission %> IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index 16c8188..8eba064 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -1,6 +1,8 @@ CREATE TYPE cdb_dataservices_client._entity_config AS ( username text, - organization_name text + organization_name text, + application_name text, + apikey_permissions json ); -- @@ -13,10 +15,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() RETURNS record AS $$ DECLARE result cdb_dataservices_client._entity_config; + apikey_config json; is_organization boolean; username text; organization_name text; BEGIN + SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; IF is_organization IS NULL THEN RAISE EXCEPTION 'User must have user configuration in the config table'; @@ -32,6 +37,8 @@ BEGIN END IF; result.username = username; result.organization_name = organization_name; + result.application_name = apikey_config->'application'; + result.apikey_permissions = apikey_config->'permissions'; RETURN result; END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; From d43c9461b1c7e332005665fa84f11436e2886790 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Wed, 5 Sep 2018 17:36:16 +0200 Subject: [PATCH 182/265] Fixing template --- .../renderer/templates/20_public_functions.erb | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index f9696db..b7546ea 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -9,23 +9,17 @@ RETURNS <%= return_type %> AS $$ DECLARE <%= return_declaration if not multi_row %> <%= user_org_declaration %> - <% unless superuser_function? -%> - appname TEXT; - apikey_permissions JSON; - <% end %> + <% unless superuser_function? -%>appname text; + apikey_permissions json;<% end %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - <% unless superuser_function? -%> - SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); - <% if requires_permission %> - IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN + <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + <% if requires_permission %>IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; - <% else %> - SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text); - <% end %><% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From a09d18a6ee01dd741a254562498667eed08b7e14 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Wed, 5 Sep 2018 17:51:08 +0200 Subject: [PATCH 183/265] Fixing template --- client/renderer/templates/20_public_functions.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index b7546ea..a085bbd 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -19,7 +19,7 @@ BEGIN <% if requires_permission %>IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; - <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %><% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From 1eff219883b9a88a7faadb1577eded53ff3c18f8 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Wed, 5 Sep 2018 17:57:16 +0200 Subject: [PATCH 184/265] Changes in _cdb_entity_config --- client/sql/20_table_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/sql/20_table_augmentation.sql b/client/sql/20_table_augmentation.sql index 18f0d77..a0a0c2f 100644 --- a/client/sql/20_table_augmentation.sql +++ b/client/sql/20_table_augmentation.sql @@ -18,7 +18,7 @@ BEGIN SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument'; @@ -62,7 +62,7 @@ BEGIN SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument'; From 17dc3ad1c7dd6d9a438ead41c295476605be9620 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 10:01:01 +0200 Subject: [PATCH 185/265] Updated exception safe private functions --- .../templates/25_exception_safe_private_functions.erb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index 863bbf4..c431aa5 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -14,7 +14,11 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %> + <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + <% if requires_permission %>IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN + RAISE EXCEPTION '<%= permission_error %>'; + END IF; + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From 393da56964c29601068f523c7ad5497aa0727af7 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 10:06:33 +0200 Subject: [PATCH 186/265] Updated exception safe private functions --- .../renderer/templates/25_exception_safe_private_functions.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index c431aa5..a3aeb9f 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -10,6 +10,8 @@ DECLARE _returned_sqlstate TEXT; _message_text TEXT; _pg_exception_context TEXT; + <% unless superuser_function? -%>appname text; + apikey_permissions json;<% end %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; From 8fb95242139f0e1bd3e78f33f055fbc1e610375b Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 10:42:56 +0200 Subject: [PATCH 187/265] Added exceptions to bulk geocoding client --- client/sql/21_bulk_geocoding_functions.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 5af493b..210c584 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -13,7 +13,24 @@ DECLARE MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; temp_table_name text; + username text; + orgname text; + appname text; + apikey_permissions json; BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + IF NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding is not allowed'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi From 56c975fef1be0211b56a95895fb31bc492bb63eb Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 10:55:34 +0200 Subject: [PATCH 188/265] Added appname parameter --- client/renderer/sql-template-renderer | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index 23bd1d1..a25969a 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -61,6 +61,9 @@ class SqlTemplateRenderer if with_user_org parameters << { 'name' => 'username', 'type' => 'text' } parameters << { 'name' => 'orgname', 'type' => 'text' } + if requires_permission + parameters << { 'name' => 'appname', 'type' => 'text' } + end end parameters + @function_signature['params'].reject(&:empty?) end From 7a7ccbbfcff9a3f1f4ce9a85839dd10c8f479fff Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 12:04:11 +0200 Subject: [PATCH 189/265] Added appname parameter to the public server functions --- .../sql/105_route_between_points.sql | 2 ++ server/extension/sql/20_geocode_street.sql | 12 ++++----- .../extension/sql/21_bulk_geocode_street.sql | 2 +- server/extension/sql/30_admin0.sql | 2 +- server/extension/sql/40_admin1.sql | 4 +-- server/extension/sql/50_namedplaces.sql | 6 ++--- server/extension/sql/60_postalcodes.sql | 24 ++++++++--------- server/extension/sql/70_ips.sql | 2 +- server/extension/sql/85_isodistance.sql | 26 +++++++++---------- server/extension/sql/90_isochrone.sql | 26 +++++++++---------- 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 87769c9..24fc7ea 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -1,6 +1,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( username TEXT, orgname TEXT, + appname TEXT, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, @@ -43,6 +44,7 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( username TEXT, orgname TEXT, + appname TEXT, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 30a477a..642f8e1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -1,6 +1,6 @@ -- Geocodes a street address given a searchtext and a state and/or country -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger,LoggerConfig @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -51,7 +51,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -66,7 +66,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -79,7 +79,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -92,7 +92,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index cee0539..f6d085b 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -5,7 +5,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 7545802..a1f3496 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname TEXT, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index f5c3a71..daa381b 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -1,5 +1,5 @@ ---- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname TEXT, admin1_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -37,7 +37,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname TEXT, admin1_name text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 55acc4e..7a06997 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -1,5 +1,5 @@ ---- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig @@ -19,7 +19,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text, country_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig @@ -39,7 +39,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 6fc27c3..f40f807 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -34,13 +34,13 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code double precision) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, appname, code::text); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code text, country text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -76,13 +76,13 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code double precision, country text) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, appname, code::text, country); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -118,14 +118,14 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code double precision) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, appname, code::text) $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code text, country text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -161,9 +161,9 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code double precision, country text) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, appname, code::text, country); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 6b361c1..97f935c 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname TEXT, ip text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 9609c27..bbce02e 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -18,23 +18,23 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, appname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- heremaps isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -49,7 +49,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapzen isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -63,7 +63,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapbox isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -77,7 +77,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- tomtom isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 61a5fce..725cf85 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -18,23 +18,23 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, appname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, appname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- heremaps isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -49,7 +49,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapzen isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapbox isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -75,7 +75,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- tomtom isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] From eca4b42a738c81538eaa892b25bddbfbe9e3cba5 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 12:32:48 +0200 Subject: [PATCH 190/265] Fixed tests --- server/extension/sql/50_namedplaces.sql | 4 +- .../test/expected/20_street_test.out | 2 +- .../test/expected/30_admin0_test.out | 6 +-- .../test/expected/40_admin1_test.out | 12 +++--- .../test/expected/50_namedplaces_test.out | 40 +++++++++---------- server/extension/test/sql/20_street_test.sql | 2 +- server/extension/test/sql/30_admin0_test.sql | 6 +-- server/extension/test/sql/40_admin1_test.sql | 12 +++--- .../test/sql/50_namedplaces_test.sql | 34 ++++++++-------- 9 files changed, 59 insertions(+), 59 deletions(-) diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 7a06997..14975b5 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -9,8 +9,8 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, appname, city_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) diff --git a/server/extension/test/expected/20_street_test.out b/server/extension/test/expected/20_street_test.out index 8fbce7a..3f02e6e 100644 --- a/server/extension/test/expected/20_street_test.out +++ b/server/extension/test/expected/20_street_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/30_admin0_test.out b/server/extension/test/expected/30_admin0_test.out index 55c7639..61fb151 100644 --- a/server/extension/test/expected/30_admin0_test.out +++ b/server/extension/test/expected/30_admin0_test.out @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); cdb_geocode_admin0_polygon ---------------------------- @@ -16,7 +16,7 @@ INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( -71.1031880899493 42.3152774590236))',4326) ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); cdb_geocode_admin0_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 @@ -28,7 +28,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/40_admin1_test.out b/server/extension/test/expected/40_admin1_test.out index 4894b0c..80a76ed 100644 --- a/server/extension/test/expected/40_admin1_test.out +++ b/server/extension/test/expected/40_admin1_test.out @@ -1,12 +1,12 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); cdb_geocode_admin1_polygon ---------------------------- (1 row) -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); cdb_geocode_admin1_polygon ---------------------------- @@ -22,13 +22,13 @@ INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['c -71.1031880899493 42.3152774590236))',4326) ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); cdb_geocode_admin1_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); cdb_geocode_admin1_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 @@ -40,7 +40,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t @@ -51,7 +51,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index 2e41285..a8e4ee9 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,20 +1,20 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ @@ -41,35 +41,35 @@ WARNING: Error geocoding namedplace using geocode street point, falling back to 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- @@ -77,17 +77,6 @@ WARNING: Error geocoding namedplace using geocode street point, falling back to (1 row) -- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) @@ -104,7 +93,18 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); exists -------- t diff --git a/server/extension/test/sql/20_street_test.sql b/server/extension/test/sql/20_street_test.sql index 86e0368..1843679 100644 --- a/server/extension/test/sql/20_street_test.sql +++ b/server/extension/test/sql/20_street_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text, text'); diff --git a/server/extension/test/sql/30_admin0_test.sql b/server/extension/test/sql/30_admin0_test.sql index 3249c60..7a558ea 100644 --- a/server/extension/test/sql/30_admin0_test.sql +++ b/server/extension/test/sql/30_admin0_test.sql @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); -- Insert some dummy synonym INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); @@ -14,7 +14,7 @@ INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); -- Check for admin0 signatures SELECT exists(SELECT * @@ -22,7 +22,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/40_admin1_test.sql b/server/extension/test/sql/40_admin1_test.sql index 7b3748c..d8c963a 100644 --- a/server/extension/test/sql/40_admin1_test.sql +++ b/server/extension/test/sql/40_admin1_test.sql @@ -1,7 +1,7 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); -- Insert dummy data into country decoder table INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); @@ -15,8 +15,8 @@ INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['c ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); -- Check for admin1 signatures SELECT exists(SELECT * @@ -24,14 +24,14 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/50_namedplaces_test.sql b/server/extension/test/sql/50_namedplaces_test.sql index 0fde77f..dd19a88 100644 --- a/server/extension/test/sql/50_namedplaces_test.sql +++ b/server/extension/test/sql/50_namedplaces_test.sql @@ -1,8 +1,8 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); -- Insert dummy data into points table INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( @@ -21,21 +21,14 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'valencia', 'Spain'); -- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) @@ -48,7 +41,14 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p From 30e480e545813654f07e5056961000c1389bb630 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 12:46:12 +0200 Subject: [PATCH 191/265] Fixed m04r tests --- .../test/expected/60_postalcodes_test.out | 36 +++++++++---------- .../extension/test/expected/70_ips_test.out | 6 ++-- .../test/expected/85_isodistance_test.out | 2 +- .../test/expected/90_isochrone_test.out | 2 +- .../expected/95_route_between_points_test.out | 4 +-- .../test/sql/60_postalcodes_test.sql | 28 +++++++-------- server/extension/test/sql/70_ips_test.sql | 6 ++-- .../test/sql/85_isodistance_test.sql | 2 +- .../extension/test/sql/90_isochrone_test.sql | 2 +- .../test/sql/95_route_between_points_test.sql | 4 +-- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/server/extension/test/expected/60_postalcodes_test.out b/server/extension/test/expected/60_postalcodes_test.out index b2a5c91..f671d79 100644 --- a/server/extension/test/expected/60_postalcodes_test.out +++ b/server/extension/test/expected/60_postalcodes_test.out @@ -6,7 +6,7 @@ DELETE FROM available_services; DELETE FROM admin0_synonyms; -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); cdb_geocode_postalcode_point ------------------------------ @@ -42,25 +42,25 @@ INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( 3 ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); cdb_geocode_postalcode_point ---------------------------------------------------- 0101000020E61000000000000000E040408036B47414764840 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204', 'spain'); cdb_geocode_postalcode_point ---------------------------------------------------- 0101000020E61000000000000000E040408036B47414764840 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204'); cdb_geocode_postalcode_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204', 'spain'); cdb_geocode_postalcode_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 @@ -78,7 +78,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t @@ -89,18 +89,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); exists -------- t @@ -117,6 +106,17 @@ SELECT exists(SELECT * t (1 row) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + exists +-------- + t +(1 row) + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/expected/70_ips_test.out b/server/extension/test/expected/70_ips_test.out index 48012f7..f5ea404 100644 --- a/server/extension/test/expected/70_ips_test.out +++ b/server/extension/test/expected/70_ips_test.out @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); cdb_geocode_ipaddress_point ----------------------------- @@ -9,7 +9,7 @@ SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_or -- Insert dummy data into ip_address_locations INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); cdb_geocode_ipaddress_point ---------------------------------------------------- 0101000020E61000003333333333334440AE47E17A14AE0D40 @@ -21,7 +21,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/85_isodistance_test.out b/server/extension/test/expected/85_isodistance_test.out index 5705284..2ecbb88 100644 --- a/server/extension/test/expected/85_isodistance_test.out +++ b/server/extension/test/expected/85_isodistance_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); exists -------- t diff --git a/server/extension/test/expected/90_isochrone_test.out b/server/extension/test/expected/90_isochrone_test.out index 7ee0f04..193b59f 100644 --- a/server/extension/test/expected/90_isochrone_test.out +++ b/server/extension/test/expected/90_isochrone_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); exists -------- t diff --git a/server/extension/test/expected/95_route_between_points_test.out b/server/extension/test/expected/95_route_between_points_test.out index 176059f..cfc8585 100644 --- a/server/extension/test/expected/95_route_between_points_test.out +++ b/server/extension/test/expected/95_route_between_points_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_point_to_point' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, geometry, text, text[], text'); exists -------- t @@ -16,7 +16,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_with_waypoints' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry[], text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry[], text, text[], text'); exists -------- t diff --git a/server/extension/test/sql/60_postalcodes_test.sql b/server/extension/test/sql/60_postalcodes_test.sql index 55d1c85..eea2c8f 100644 --- a/server/extension/test/sql/60_postalcodes_test.sql +++ b/server/extension/test/sql/60_postalcodes_test.sql @@ -7,7 +7,7 @@ DELETE FROM admin0_synonyms; -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); -- Insert dummy data into ip_address_locations INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( @@ -44,13 +44,13 @@ INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204', 'spain'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204', 'spain'); -- Clean dbs DELETE FROM global_postal_code_points; @@ -65,21 +65,14 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p @@ -88,6 +81,13 @@ SELECT exists(SELECT * AND proname = 'cdb_geocode_postalcode_polygon' AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/70_ips_test.sql b/server/extension/test/sql/70_ips_test.sql index 6111bd0..79bea61 100644 --- a/server/extension/test/sql/70_ips_test.sql +++ b/server/extension/test/sql/70_ips_test.sql @@ -1,12 +1,12 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); -- Insert dummy data into ip_address_locations INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); -- Check for namedplaces signatures (point and polygon) SELECT exists(SELECT * @@ -14,7 +14,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/85_isodistance_test.sql b/server/extension/test/sql/85_isodistance_test.sql index f66b68a..4f5c37f 100644 --- a/server/extension/test/sql/85_isodistance_test.sql +++ b/server/extension/test/sql/85_isodistance_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/sql/90_isochrone_test.sql b/server/extension/test/sql/90_isochrone_test.sql index 113fb28..60819bc 100644 --- a/server/extension/test/sql/90_isochrone_test.sql +++ b/server/extension/test/sql/90_isochrone_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/sql/95_route_between_points_test.sql b/server/extension/test/sql/95_route_between_points_test.sql index 4acc15d..21fe257 100644 --- a/server/extension/test/sql/95_route_between_points_test.sql +++ b/server/extension/test/sql/95_route_between_points_test.sql @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_point_to_point' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, geometry, text, text[], text'); -- Check for routing waypoint route signatures SELECT exists(SELECT * @@ -12,4 +12,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_with_waypoints' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry[], text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry[], text, text[], text'); From 04a5a25d062003069b54c4ce9f38300da5d05495 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 12:55:31 +0200 Subject: [PATCH 192/265] Fixed m04r tests --- server/extension/test/expected/50_namedplaces_test.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index a8e4ee9..cab8f50 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -34,7 +34,7 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E -- Insert dummy data into admin1 decoder table INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- From 603d691017795b6ca0fdfc7dfe5e976ba5c7d223 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 13:47:50 +0200 Subject: [PATCH 193/265] Added metrics --- .../sql/105_route_between_points.sql | 4 +-- server/extension/sql/20_geocode_street.sql | 2 +- .../extension/sql/21_bulk_geocode_street.sql | 2 +- server/extension/sql/30_admin0.sql | 5 +++- server/extension/sql/40_admin1.sql | 4 ++- server/extension/sql/50_namedplaces.sql | 26 ++++++++++--------- server/extension/sql/60_postalcodes.sql | 5 +++- server/extension/sql/70_ips.sql | 3 +++ server/extension/sql/85_isodistance.sql | 2 +- server/extension/sql/90_isochrone.sql | 2 +- 10 files changed, 34 insertions(+), 21 deletions(-) diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 24fc7ea..e6cede3 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -19,7 +19,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} with metrics('cdb_route_with_point', user_routing_config, logger, params): waypoints = [origin, destination] @@ -61,7 +61,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 642f8e1..30155a1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -12,7 +12,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.heremaps_geocoder: diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index f6d085b..934abe3 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -20,7 +20,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'searches': searches} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'searches': searches} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index a1f3496..6e670ab 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -14,7 +14,10 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + + params = {'username': username, 'orgname': orgname, 'appname': appname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [country_name], 1) diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index daa381b..99e28b4 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -16,7 +16,9 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + params = {'username': username, 'orgname': orgname, 'appname': appname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [admin1_name], 1) diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 14975b5..07a18f0 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -14,8 +14,8 @@ RETURNS Geometry AS $$ except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, appname, city_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) @@ -29,13 +29,13 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4, NULL, NULL, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, appname, city_name, country_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, NULL, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, appname, city_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) @@ -49,16 +49,16 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4, NULL, $5, $6) as point;", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, appname, city_name, admin1_name, country_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5, $6) as point;", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, appname, city_name, admin1_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, appname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig, metrics @@ -74,7 +74,9 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + params = {'username': username, 'orgname': orgname, 'appname': appname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): try: if admin1_name and country_name: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index f40f807..8390e71 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -14,7 +14,10 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + + params = {'username': username, 'orgname': orgname, 'appname': appname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) rv = plpy.execute(plan, [code], 1) diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 97f935c..7eb16e3 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -14,6 +14,9 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'appname': appname, 'ip': ip} + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index bbce02e..3fb7feb 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -14,7 +14,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - params = {'source': source, 'mode': mode, 'range': range, 'options': options} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'source': source, 'mode': mode, 'range': range, 'options': options} with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 725cf85..5a06268 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -14,7 +14,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - params = {'source': source, 'mode': mode, 'range': range, 'options': options} + params = {'username': username, 'orgname': orgname, 'appname': appname, 'source': source, 'mode': mode, 'range': range, 'options': options} with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: From 8a8970efa8dcf03acf8a5f0514a8f8f73f036f6c Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 15:18:42 +0200 Subject: [PATCH 194/265] Fixed tests --- client/test/expected/10_admin0_test.out | 6 +-- client/test/expected/20_admin1_test.out | 12 ++--- ..._exception_safe_private_functions_test.out | 12 ++--- client/test/expected/30_namedplaces_test.out | 18 ++++---- client/test/expected/40_postalcodes_test.out | 12 ++--- client/test/expected/50_ipaddresses_test.out | 6 +-- client/test/expected/60_street_test.out | 24 +++++----- .../expected/80_route_point_to_point_test.out | 20 ++++----- client/test/expected/999_permissions_test.out | 44 +++++++++---------- client/test/sql/10_admin0_test.sql | 4 +- client/test/sql/20_admin1_test.sql | 8 ++-- ..._exception_safe_private_functions_test.sql | 6 +-- client/test/sql/30_namedplaces_test.sql | 12 ++--- client/test/sql/40_postalcodes_test.sql | 8 ++-- client/test/sql/50_ipaddresses_test.sql | 4 +- client/test/sql/60_street_test.sql | 4 +- .../test/sql/80_route_point_to_point_test.sql | 8 ++-- client/test/sql/999_permissions_test.sql | 22 +++++----- 18 files changed, 115 insertions(+), 115 deletions(-) diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 02e3351..039f67c 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -2,16 +2,16 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server function -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %, %)', username, orgname, appname, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , , Spain) cdb_geocode_admin0_polygon ---------------------------- diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index 0ead0aa..a34b87c 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -2,30 +2,30 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, appname, admin1_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California, United States) cdb_geocode_admin1_polygon ---------------------------- diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index aeb5bd9..94e9afe 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -2,20 +2,20 @@ SET client_min_messages TO warning; SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions to raise exceptions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, appname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF isoline AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; @@ -31,20 +31,20 @@ $$ LANGUAGE 'plpgsql'; SET ROLE test_regular_user; -- Exercise the exception safe and the proxied functions SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); -WARNING: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. _cdb_geocode_street_point_exception_safe ------------------------------------------ (1 row) SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); -WARNING: cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_isodistance(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. center | data_range | the_geom --------+------------+---------- (0 rows) SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -WARNING: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. shape | length | duration -------+--------+---------- | | diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index 2460f2b..d92e452 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -2,44 +2,44 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, appname, city_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, appname, city_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %, %)', username, orgname, appname, city_name, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia, Spain) cdb_geocode_namedplace_point ------------------------------ diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index 4578fd4..9efe91b 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -2,30 +2,30 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , , 03204, Spain) cdb_geocode_postalcode_polygon -------------------------------- (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , , 03204, Spain) cdb_geocode_postalcode_point ------------------------------ diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index 30bf75e..69bcd94 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -2,16 +2,16 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname text, ip_address text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %, %)', username, orgname, appname, ip_address; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , , 8.8.8.8) cdb_geocode_ipaddress_point ----------------------------- diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index 7955eeb..42d45eb 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -2,79 +2,79 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, searchtext, city, state_province, country; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , country) cdb_geocode_street_point -------------------------- diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index e6b51d3..46c59c1 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -2,64 +2,64 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %, %)', username, orgname, appname, origin, destination, mode, options, units; SELECT NULL, 5.33, 100 INTO ret; RETURN ret; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, appname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %)', username, orgname, waypoints, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, waypoints, mode, options, units; SELECT NULL, 2.22, 500 INTO ret; RETURN ret; END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied functions SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) cdb_route_with_waypoints -------------------------- (,2.22,500) diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index a69a009..e74ec7f 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -6,102 +6,102 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Exercise the public function -- it is public, it shall work SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , , Spain) cdb_geocode_admin0_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California, United States) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , , 03204, Spain) cdb_geocode_postalcode_polygon -------------------------------- (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , , 03204, Spain) cdb_geocode_postalcode_point ------------------------------ (1 row) SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , , 8.8.8.8) cdb_geocode_ipaddress_point ----------------------------- (1 row) SELECT cdb_geocode_street_point('one street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , one street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , one street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) -- Check the regular user has no permissions on private functions -SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin0_polygon -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin1_polygon -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin1_polygon -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell', 'Ugly world'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); ERROR: permission denied for function _cdb_geocode_postalcode_polygon -SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); ERROR: permission denied for function _cdb_geocode_postalcode_point -SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', 'evil_appname', '8.8.8.8'); ERROR: permission denied for function _cdb_geocode_ipaddress_point -SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'one street, 1'); ERROR: permission denied for function _cdb_geocode_street_point -SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); ERROR: permission denied for function _cdb_route_point_to_point diff --git a/client/test/sql/10_admin0_test.sql b/client/test/sql/10_admin0_test.sql index bc0f005..182ec3d 100644 --- a/client/test/sql/10_admin0_test.sql +++ b/client/test/sql/10_admin0_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server function -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %, %)', username, orgname, appname, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/20_admin1_test.sql b/client/test/sql/20_admin1_test.sql index 458b6c3..ce02a0c 100644 --- a/client/test/sql/20_admin1_test.sql +++ b/client/test/sql/20_admin1_test.sql @@ -3,18 +3,18 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, appname, admin1_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index 67d48df..2cf139c 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -3,7 +3,7 @@ SET client_min_messages TO warning; SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions to raise exceptions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; @@ -11,14 +11,14 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, appname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF isoline AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; diff --git a/client/test/sql/30_namedplaces_test.sql b/client/test/sql/30_namedplaces_test.sql index ed3acd3..3816072 100644 --- a/client/test/sql/30_namedplaces_test.sql +++ b/client/test/sql/30_namedplaces_test.sql @@ -3,26 +3,26 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, appname, city_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, appname, city_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %, %)', username, orgname, appname, city_name, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/40_postalcodes_test.sql b/client/test/sql/40_postalcodes_test.sql index 12b2789..5b8bafc 100644 --- a/client/test/sql/40_postalcodes_test.sql +++ b/client/test/sql/40_postalcodes_test.sql @@ -3,18 +3,18 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/50_ipaddresses_test.sql b/client/test/sql/50_ipaddresses_test.sql index 3209d8e..098ebc6 100644 --- a/client/test/sql/50_ipaddresses_test.sql +++ b/client/test/sql/50_ipaddresses_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname text, ip_address text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %, %)', username, orgname, appname, ip_address; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/60_street_test.sql b/client/test/sql/60_street_test.sql index 4e7f695..116dec9 100644 --- a/client/test/sql/60_street_test.sql +++ b/client/test/sql/60_street_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, searchtext, city, state_province, country; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/80_route_point_to_point_test.sql b/client/test/sql/80_route_point_to_point_test.sql index c63d76b..5f6c4c3 100644 --- a/client/test/sql/80_route_point_to_point_test.sql +++ b/client/test/sql/80_route_point_to_point_test.sql @@ -4,24 +4,24 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %, %)', username, orgname, appname, origin, destination, mode, options, units; SELECT NULL, 5.33, 100 INTO ret; RETURN ret; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, appname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %)', username, orgname, waypoints, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, waypoints, mode, options, units; SELECT NULL, 2.22, 500 INTO ret; RETURN ret; END; diff --git a/client/test/sql/999_permissions_test.sql b/client/test/sql/999_permissions_test.sql index adc649e..004bbf6 100644 --- a/client/test/sql/999_permissions_test.sql +++ b/client/test/sql/999_permissions_test.sql @@ -20,14 +20,14 @@ SELECT cdb_geocode_street_point('one street, 1'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -- Check the regular user has no permissions on private functions -SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); -SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); -SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); -SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); -SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); -SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', 'evil_appname', '8.8.8.8'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'one street, 1'); +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); From ca717ed123e7fae109798338b16fb5158e24f555 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 15:40:41 +0200 Subject: [PATCH 195/265] Fixed permission check --- client/renderer/templates/20_public_functions.erb | 2 +- .../renderer/templates/25_exception_safe_private_functions.erb | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index a085bbd..2b98906 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -16,7 +16,7 @@ BEGIN RAISE EXCEPTION 'The api_key must be provided'; END IF; <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); - <% if requires_permission %>IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN + <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index a3aeb9f..8d92420 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -17,7 +17,7 @@ BEGIN RAISE EXCEPTION 'The api_key must be provided'; END IF; <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); - <% if requires_permission %>IF NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN + <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 210c584..edaefba 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -22,7 +22,7 @@ BEGIN RAISE EXCEPTION 'The api_key must be provided'; END IF; SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); - IF NOT apikey_permissions::jsonb ? 'geocoding' THEN + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN RAISE EXCEPTION 'Geocoding is not allowed'; END IF; From 5ea1a4ca6be7c1bbd4481c725c031080a8604606 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 6 Sep 2018 16:40:05 +0200 Subject: [PATCH 196/265] Added permissions to tests --- client/test/expected/10_admin0_test.out | 30 ++- client/test/expected/20_admin1_test.out | 36 ++- .../21_bulk_geocoding_functions_test.out | 34 +++ ..._exception_safe_private_functions_test.out | 42 +++- client/test/expected/30_namedplaces_test.out | 42 +++- client/test/expected/40_postalcodes_test.out | 36 ++- client/test/expected/50_ipaddresses_test.out | 30 ++- client/test/expected/60_street_test.out | 84 ++++++- .../expected/80_route_point_to_point_test.out | 60 ++++- client/test/expected/999_permissions_test.out | 229 ++++++++++++------ client/test/sql/10_admin0_test.sql | 14 +- client/test/sql/20_admin1_test.sql | 16 +- .../sql/21_bulk_geocoding_functions_test.sql | 19 +- ..._exception_safe_private_functions_test.sql | 27 ++- client/test/sql/30_namedplaces_test.sql | 18 +- client/test/sql/40_postalcodes_test.sql | 17 +- client/test/sql/50_ipaddresses_test.sql | 15 +- client/test/sql/60_street_test.sql | 33 ++- .../test/sql/80_route_point_to_point_test.sql | 25 +- client/test/sql/999_permissions_test.sql | 65 +++-- 20 files changed, 732 insertions(+), 140 deletions(-) diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 039f67c..0da1802 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -9,11 +9,37 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , , Spain) +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_admin0_polygon('Spain'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , "testing_app", Spain) cdb_geocode_admin0_polygon ---------------------------- (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index a34b87c..63ee13d 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -16,18 +16,48 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California) +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California, United States) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California, United States) cdb_geocode_admin1_polygon ---------------------------- (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index c846a1d..aa1b68b 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -9,12 +9,46 @@ CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_si RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; +-- No permissions granted +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ERROR: Geocoding is not allowed +-- Test quota check by mocking quota 0 +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ERROR: Geocoding is not allowed +-- Test quota check by mocking quota 0 +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); ERROR: Remaining quota: 0. Estimated cost: 1 -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Remaining quota: 0. Estimated cost: 1 +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index 94e9afe..778b1d0 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -27,9 +27,39 @@ BEGIN RETURN ret; END; $$ LANGUAGE 'plpgsql'; --- Use regular user role -SET ROLE test_regular_user; -- Exercise the exception safe and the proxied functions +-- No permissions granted +SET ROLE test_regular_user; -- Use regular user role +SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); +ERROR: Geocoding is not allowed +SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); +ERROR: Isolines are not allowed +SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +-- No permissions granted but conf created +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": []}'); + cdb_conf_setconf +------------------ + +(1 row) + +SET ROLE test_regular_user; -- Use regular user role +SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); +ERROR: Geocoding is not allowed +SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); +ERROR: Isolines are not allowed +SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +-- Grant geocoding permissions +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); WARNING: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. _cdb_geocode_street_point_exception_safe @@ -50,3 +80,11 @@ WARNING: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regress | | (1 row) +-- Remove permissions +SET ROLE postgres; +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index d92e452..d8515b3 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -23,25 +23,59 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx) +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index 9efe91b..6ed22f7 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -16,18 +16,48 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , , 03204, Spain) +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , "testing_app", 03204, Spain) cdb_geocode_postalcode_polygon -------------------------------- (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , , 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , "testing_app", 03204, Spain) cdb_geocode_postalcode_point ------------------------------ (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index 69bcd94..54bdbb1 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -9,11 +9,37 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , , 8.8.8.8) +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , "testing_app", 8.8.8.8) cdb_geocode_ipaddress_point ----------------------------- (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index 42d45eb..adf14d4 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -9,74 +9,136 @@ BEGIN RETURN NULL; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, 1, , , ) +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street, 1'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +ERROR: Geocoding is not allowed +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_street_point('One street, 1'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street, 1'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +ERROR: Geocoding is not allowed +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_geocode_street_point('One street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , country) cdb_geocode_street_point -------------------------- (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index 46c59c1..ef2a2a6 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -22,46 +22,92 @@ BEGIN RETURN ret; END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied functions +-- -- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +ERROR: Routing is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); +ERROR: Routing is not allowed +-- Grant other permissions but routing +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +ERROR: Routing is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); +ERROR: Routing is not allowed +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); +ERROR: Routing is not allowed +-- Grant routing permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index e74ec7f..2740658 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -3,85 +3,6 @@ SET ROLE test_regular_user; -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; --- Exercise the public function --- it is public, it shall work -SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , , Spain) - cdb_geocode_admin0_polygon ----------------------------- - -(1 row) - -SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California) - cdb_geocode_admin1_polygon ----------------------------- - -(1 row) - -SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , , California, United States) - cdb_geocode_admin1_polygon ----------------------------- - -(1 row) - -SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx) - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - -SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia) - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - -SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , , Elx, Valencia, Spain) - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - -SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , , 03204, Spain) - cdb_geocode_postalcode_polygon --------------------------------- - -(1 row) - -SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , , 03204, Spain) - cdb_geocode_postalcode_point ------------------------------- - -(1 row) - -SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , , 8.8.8.8) - cdb_geocode_ipaddress_point ------------------------------ - -(1 row) - -SELECT cdb_geocode_street_point('one street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , , one street, 1, , , ) - cdb_geocode_street_point --------------------------- - -(1 row) - -SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) - cdb_route_point_to_point --------------------------- - (,5.33,100) -(1 row) - -- Check the regular user has no permissions on private functions SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin0_polygon @@ -105,3 +26,153 @@ SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'o ERROR: permission denied for function _cdb_geocode_street_point SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); ERROR: permission denied for function _cdb_route_point_to_point +-- -- Exercise the public function +-- -- it is public, it shall work +-- No permissions granted +SELECT cdb_geocode_admin0_polygon('Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('one street, 1'); +ERROR: Geocoding is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +ERROR: Routing is not allowed +-- Grant other permissions but geocoding and routing +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SET ROLE test_regular_user; -- Use regular user role +SELECT cdb_geocode_admin0_polygon('Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +ERROR: Geocoding is not allowed +SELECT cdb_geocode_street_point('one street, 1'); +ERROR: Geocoding is not allowed +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +ERROR: Routing is not allowed +-- Grant geocoding permissions +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SET ROLE test_regular_user; -- Use regular user role +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , "testing_app", Spain) + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California) + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California, United States) + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx) + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia) + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia, Spain) + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , "testing_app", 03204, Spain) + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , "testing_app", 03204, Spain) + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , "testing_app", 8.8.8.8) + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('one street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", one street, 1, , , ) + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) + cdb_route_point_to_point +-------------------------- + (,5.33,100) +(1 row) + +-- Remove permissions +SET ROLE postgres; +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/sql/10_admin0_test.sql b/client/test/sql/10_admin0_test.sql index 182ec3d..f4d24c6 100644 --- a/client/test/sql/10_admin0_test.sql +++ b/client/test/sql/10_admin0_test.sql @@ -11,6 +11,18 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +-- -- Exercise the public and the proxied function --- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_admin0_polygon('Spain'); + +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_admin0_polygon('Spain'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_admin0_polygon('Spain'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/20_admin1_test.sql b/client/test/sql/20_admin1_test.sql index ce02a0c..ebefcb9 100644 --- a/client/test/sql/20_admin1_test.sql +++ b/client/test/sql/20_admin1_test.sql @@ -19,7 +19,21 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +-- -- Exercise the public and the proxied function --- Exercise the public and the proxied function +-- No permissions granted SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); + +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index 985133a..649d8ce 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -12,12 +12,29 @@ RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; +-- No permissions granted -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); - -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +-- Test quota check by mocking quota 0 +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +-- Test bulk size not mandatory (it will get the optimal) +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +-- Test quota check by mocking quota 0 +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index 2cf139c..aeff422 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -32,11 +32,30 @@ END; $$ LANGUAGE 'plpgsql'; - --- Use regular user role -SET ROLE test_regular_user; - -- Exercise the exception safe and the proxied functions + +-- No permissions granted +SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); + +-- No permissions granted but conf created +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": []}'); +SET ROLE test_regular_user; -- Use regular user role +SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); +SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); +SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); + +-- Grant geocoding permissions +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing", "isolines"]}'); +SET ROLE test_regular_user; -- Use regular user role +SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); +SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); +SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); + +-- Remove permissions +SET ROLE postgres; +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/30_namedplaces_test.sql b/client/test/sql/30_namedplaces_test.sql index 3816072..1d1d83c 100644 --- a/client/test/sql/30_namedplaces_test.sql +++ b/client/test/sql/30_namedplaces_test.sql @@ -27,8 +27,24 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function + +-- No permissions granted SELECT cdb_geocode_namedplace_point('Elx'); SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/40_postalcodes_test.sql b/client/test/sql/40_postalcodes_test.sql index 5b8bafc..369a44b 100644 --- a/client/test/sql/40_postalcodes_test.sql +++ b/client/test/sql/40_postalcodes_test.sql @@ -19,6 +19,21 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function + +-- No permissions granted SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); + +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); \ No newline at end of file diff --git a/client/test/sql/50_ipaddresses_test.sql b/client/test/sql/50_ipaddresses_test.sql index 098ebc6..0c68dd2 100644 --- a/client/test/sql/50_ipaddresses_test.sql +++ b/client/test/sql/50_ipaddresses_test.sql @@ -12,5 +12,18 @@ END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function + +-- No permissions granted SELECT cdb_geocode_ipaddress_point('8.8.8.8'); + +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/60_street_test.sql b/client/test/sql/60_street_test.sql index 116dec9..ac0d107 100644 --- a/client/test/sql/60_street_test.sql +++ b/client/test/sql/60_street_test.sql @@ -12,7 +12,9 @@ END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied function +-- -- Exercise the public and the proxied function + +-- No permissions granted SELECT cdb_geocode_street_point('One street, 1'); SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); @@ -23,3 +25,32 @@ SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); + +-- Grant other permissions but geocoding +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); + +-- Grant geocoding permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); \ No newline at end of file diff --git a/client/test/sql/80_route_point_to_point_test.sql b/client/test/sql/80_route_point_to_point_test.sql index 5f6c4c3..a995425 100644 --- a/client/test/sql/80_route_point_to_point_test.sql +++ b/client/test/sql/80_route_point_to_point_test.sql @@ -28,12 +28,33 @@ END; $$ LANGUAGE 'plpgsql'; --- Exercise the public and the proxied functions +-- -- Exercise the public and the proxied function + +-- No permissions granted SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); - SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); +-- Grant other permissions but routing +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "isolines"]}'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); + +-- Grant routing permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing"]}'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); +SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); + +-- Remove permissions +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/999_permissions_test.sql b/client/test/sql/999_permissions_test.sql index 004bbf6..0417656 100644 --- a/client/test/sql/999_permissions_test.sql +++ b/client/test/sql/999_permissions_test.sql @@ -5,20 +5,6 @@ SET ROLE test_regular_user; -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; --- Exercise the public function --- it is public, it shall work -SELECT cdb_geocode_admin0_polygon('Spain'); -SELECT cdb_geocode_admin1_polygon('California'); -SELECT cdb_geocode_admin1_polygon('California', 'United States'); -SELECT cdb_geocode_namedplace_point('Elx'); -SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -SELECT cdb_geocode_street_point('one street, 1'); -SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); - -- Check the regular user has no permissions on private functions SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); @@ -31,3 +17,54 @@ SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', 'evil_appname' SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', 'evil_appname', '8.8.8.8'); SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'one street, 1'); SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); + +-- -- Exercise the public function +-- -- it is public, it shall work +-- No permissions granted +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point('one street, 1'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); + +-- Grant other permissions but geocoding and routing +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["isolines"]}'); +SET ROLE test_regular_user; -- Use regular user role +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point('one street, 1'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); + +-- Grant geocoding permissions +SET ROLE postgres; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing"]}'); +SET ROLE test_regular_user; -- Use regular user role +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point('one street, 1'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); + +-- Remove permissions +SET ROLE postgres; +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); From 4b6b4e92e8ff065f33ca93f78725191ab06fca87 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 12:54:37 +0200 Subject: [PATCH 197/265] CR suggestions (remove appname) --- client/renderer/sql-template-renderer | 3 -- .../templates/20_public_functions.erb | 7 ++-- .../25_exception_safe_private_functions.erb | 5 ++- client/sql/15_config_management.sql | 2 -- client/sql/21_bulk_geocoding_functions.sql | 3 +- client/test/expected/10_admin0_test.out | 4 +-- client/test/expected/20_admin1_test.out | 8 ++--- ..._exception_safe_private_functions_test.out | 6 ++-- client/test/expected/30_namedplaces_test.out | 12 +++---- client/test/expected/40_postalcodes_test.out | 8 ++--- client/test/expected/50_ipaddresses_test.out | 4 +-- client/test/expected/60_street_test.out | 4 +-- .../expected/80_route_point_to_point_test.out | 8 ++--- client/test/expected/999_permissions_test.out | 22 ++++++------ client/test/sql/10_admin0_test.sql | 4 +-- client/test/sql/20_admin1_test.sql | 8 ++--- ..._exception_safe_private_functions_test.sql | 6 ++-- client/test/sql/30_namedplaces_test.sql | 12 +++---- client/test/sql/40_postalcodes_test.sql | 8 ++--- client/test/sql/50_ipaddresses_test.sql | 4 +-- client/test/sql/60_street_test.sql | 4 +-- .../test/sql/80_route_point_to_point_test.sql | 8 ++--- client/test/sql/999_permissions_test.sql | 22 ++++++------ .../sql/105_route_between_points.sql | 6 ++-- server/extension/sql/20_geocode_street.sql | 14 ++++---- .../extension/sql/21_bulk_geocode_street.sql | 4 +-- server/extension/sql/30_admin0.sql | 4 +-- server/extension/sql/40_admin1.sql | 6 ++-- server/extension/sql/50_namedplaces.sql | 34 +++++++++---------- server/extension/sql/60_postalcodes.sql | 26 +++++++------- server/extension/sql/70_ips.sql | 4 +-- server/extension/sql/85_isodistance.sql | 20 +++++------ server/extension/sql/90_isochrone.sql | 20 +++++------ .../test/expected/30_admin0_test.out | 4 +-- .../test/expected/40_admin1_test.out | 8 ++--- .../test/expected/50_namedplaces_test.out | 18 +++++----- .../extension/test/expected/70_ips_test.out | 4 +-- server/extension/test/sql/30_admin0_test.sql | 4 +-- server/extension/test/sql/40_admin1_test.sql | 8 ++--- .../test/sql/50_namedplaces_test.sql | 18 +++++----- server/extension/test/sql/70_ips_test.sql | 4 +-- 41 files changed, 184 insertions(+), 194 deletions(-) diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index a25969a..23bd1d1 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -61,9 +61,6 @@ class SqlTemplateRenderer if with_user_org parameters << { 'name' => 'username', 'type' => 'text' } parameters << { 'name' => 'orgname', 'type' => 'text' } - if requires_permission - parameters << { 'name' => 'appname', 'type' => 'text' } - end end parameters + @function_signature['params'].reject(&:empty?) end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 2b98906..3950037 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -9,17 +9,16 @@ RETURNS <%= return_type %> AS $$ DECLARE <%= return_declaration if not multi_row %> <%= user_org_declaration %> - <% unless superuser_function? -%>appname text; - apikey_permissions json;<% end %> + <% unless superuser_function? -%>apikey_permissions json;<% end %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; - <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index 8d92420..0c5cdb2 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -10,13 +10,12 @@ DECLARE _returned_sqlstate TEXT; _message_text TEXT; _pg_exception_context TEXT; - <% unless superuser_function? -%>appname text; - apikey_permissions json;<% end %> + <% unless superuser_function? -%>apikey_permissions json;<% end %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - <% unless superuser_function? -%>SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index 8eba064..d1e92f5 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -1,7 +1,6 @@ CREATE TYPE cdb_dataservices_client._entity_config AS ( username text, organization_name text, - application_name text, apikey_permissions json ); @@ -37,7 +36,6 @@ BEGIN END IF; result.username = username; result.organization_name = organization_name; - result.application_name = apikey_config->'application'; result.apikey_permissions = apikey_config->'permissions'; RETURN result; END; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index edaefba..5fd9284 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -15,13 +15,12 @@ DECLARE temp_table_name text; username text; orgname text; - appname text; apikey_permissions json; BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o, a, p INTO username, orgname, appname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN RAISE EXCEPTION 'Geocoding is not allowed'; END IF; diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 0da1802..190fef3 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -2,10 +2,10 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server function -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %, %)', username, orgname, appname, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index 63ee13d..590fb8f 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -2,17 +2,17 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, appname, admin1_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index 778b1d0..65d9df7 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -2,20 +2,20 @@ SET client_min_messages TO warning; SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions to raise exceptions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, appname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF isoline AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index d8515b3..1936f54 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -2,24 +2,24 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, appname, city_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, appname, city_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %, %)', username, orgname, appname, city_name, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index 6ed22f7..5af322f 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -2,17 +2,17 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index 54bdbb1..fac1696 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -2,10 +2,10 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname text, ip_address text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %, %)', username, orgname, appname, ip_address; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index adf14d4..2289058 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -2,10 +2,10 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, searchtext, city, state_province, country; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index ef2a2a6..2984bbf 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -2,22 +2,22 @@ -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %, %)', username, orgname, appname, origin, destination, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; SELECT NULL, 5.33, 100 INTO ret; RETURN ret; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, appname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, waypoints, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %)', username, orgname, waypoints, mode, options, units; SELECT NULL, 2.22, 500 INTO ret; RETURN ret; END; diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index 2740658..c359e60 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -4,27 +4,27 @@ SET ROLE test_regular_user; -- Add to the search path the schema SET search_path TO public,cartodb,cdb_dataservices_client; -- Check the regular user has no permissions on private functions -SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin0_polygon -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin1_polygon -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); ERROR: permission denied for function _cdb_geocode_admin1_polygon -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); ERROR: permission denied for function _cdb_geocode_namedplace_point -SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); ERROR: permission denied for function _cdb_geocode_postalcode_polygon -SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); ERROR: permission denied for function _cdb_geocode_postalcode_point -SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', 'evil_appname', '8.8.8.8'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); ERROR: permission denied for function _cdb_geocode_ipaddress_point -SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'one street, 1'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); ERROR: permission denied for function _cdb_geocode_street_point -SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); ERROR: permission denied for function _cdb_route_point_to_point -- -- Exercise the public function -- -- it is public, it shall work diff --git a/client/test/sql/10_admin0_test.sql b/client/test/sql/10_admin0_test.sql index f4d24c6..f5e1bd0 100644 --- a/client/test/sql/10_admin0_test.sql +++ b/client/test/sql/10_admin0_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server function -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %, %)', username, orgname, appname, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/20_admin1_test.sql b/client/test/sql/20_admin1_test.sql index ebefcb9..fd2250d 100644 --- a/client/test/sql/20_admin1_test.sql +++ b/client/test/sql/20_admin1_test.sql @@ -3,18 +3,18 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, appname, admin1_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index aeff422..45dbfca 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -3,7 +3,7 @@ SET client_min_messages TO warning; SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions to raise exceptions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; @@ -11,14 +11,14 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, appname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF isoline AS $$ BEGIN RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.'; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; diff --git a/client/test/sql/30_namedplaces_test.sql b/client/test/sql/30_namedplaces_test.sql index 1d1d83c..e410a9c 100644 --- a/client/test/sql/30_namedplaces_test.sql +++ b/client/test/sql/30_namedplaces_test.sql @@ -3,26 +3,26 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, appname, city_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, appname, city_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname text, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %, %)', username, orgname, appname, city_name, admin1_name, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/40_postalcodes_test.sql b/client/test/sql/40_postalcodes_test.sql index 369a44b..ffb2d1b 100644 --- a/client/test/sql/40_postalcodes_test.sql +++ b/client/test/sql/40_postalcodes_test.sql @@ -3,18 +3,18 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname text, postal_code text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %, %)', username, orgname, appname, postal_code, country_name; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/50_ipaddresses_test.sql b/client/test/sql/50_ipaddresses_test.sql index 0c68dd2..901e21b 100644 --- a/client/test/sql/50_ipaddresses_test.sql +++ b/client/test/sql/50_ipaddresses_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname text, ip_address text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %, %)', username, orgname, appname, ip_address; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/60_street_test.sql b/client/test/sql/60_street_test.sql index ac0d107..17a7a61 100644 --- a/client/test/sql/60_street_test.sql +++ b/client/test/sql/60_street_test.sql @@ -3,10 +3,10 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, appname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, searchtext, city, state_province, country; + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; RETURN NULL; END; $$ LANGUAGE 'plpgsql'; diff --git a/client/test/sql/80_route_point_to_point_test.sql b/client/test/sql/80_route_point_to_point_test.sql index a995425..fcc04aa 100644 --- a/client/test/sql/80_route_point_to_point_test.sql +++ b/client/test/sql/80_route_point_to_point_test.sql @@ -4,24 +4,24 @@ SET search_path TO public,cartodb,cdb_dataservices_client; -- Mock the server functions -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, appname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %, %)', username, orgname, appname, origin, destination, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; SELECT NULL, 5.33, 100 INTO ret; RETURN ret; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, appname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(username text, orgname text, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_client.simple_route AS $$ DECLARE ret cdb_dataservices_client.simple_route; BEGIN - RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %, %)', username, orgname, appname, waypoints, mode, options, units; + RAISE NOTICE 'cdb_dataservices_server.cdb_route_with_waypoints invoked with params (%, %, %, %, %, %)', username, orgname, waypoints, mode, options, units; SELECT NULL, 2.22, 500 INTO ret; RETURN ret; END; diff --git a/client/test/sql/999_permissions_test.sql b/client/test/sql/999_permissions_test.sql index 0417656..3adb266 100644 --- a/client/test/sql/999_permissions_test.sql +++ b/client/test/sql/999_permissions_test.sql @@ -6,17 +6,17 @@ SET ROLE test_regular_user; SET search_path TO public,cartodb,cdb_dataservices_client; -- Check the regular user has no permissions on private functions -SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Hell'); -SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell'); -SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'evil_appname', 'Sheol', 'Hell', 'Ugly world'); -SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); -SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', 'evil_appname', '66666', 'Hell'); -SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', 'evil_appname', '8.8.8.8'); -SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'evil_appname', 'one street, 1'); -SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'evil_appname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -- -- Exercise the public function -- -- it is public, it shall work diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index e6cede3..703e0dd 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -1,7 +1,6 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( username TEXT, orgname TEXT, - appname TEXT, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, @@ -19,7 +18,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} with metrics('cdb_route_with_point', user_routing_config, logger, params): waypoints = [origin, destination] @@ -44,7 +43,6 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( username TEXT, orgname TEXT, - appname TEXT, waypoints geometry(Point, 4326)[], mode TEXT, options text[] DEFAULT ARRAY[]::text[], @@ -61,7 +59,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 30155a1..882d5e8 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -1,6 +1,6 @@ -- Geocodes a street address given a searchtext and a state and/or country -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger,LoggerConfig @@ -12,7 +12,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.heremaps_geocoder: @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -51,7 +51,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -66,7 +66,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -79,7 +79,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -92,7 +92,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 934abe3..173e684 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -5,7 +5,7 @@ CREATE TYPE cdb_dataservices_server.geocoding AS ( metadata jsonb ); -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, appname TEXT, searches jsonb) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -20,7 +20,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'searches': searches} + params = {'username': username, 'orgname': orgname, 'searches': searches} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 6e670ab..c5d8c8e 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, appname TEXT, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig @@ -15,7 +15,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'country_name': country_name} + params = {'username': username, 'orgname': orgname, 'country_name': country_name} with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): try: diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 99e28b4..c10e3b2 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -1,5 +1,5 @@ ---- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname TEXT, admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -16,7 +16,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'admin1_name': admin1_name} + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): try: @@ -39,7 +39,7 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, appname TEXT, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 07a18f0..932fc78 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -1,5 +1,5 @@ ---- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig @@ -9,17 +9,17 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, appname, city_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4) as point;", ["text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, appname, city_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig @@ -29,17 +29,17 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4, NULL, NULL, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, appname, city_name, country_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, NULL, $5) as point;", ["text", "text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, appname, city_name, country_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, appname TEXT, city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) RETURNS Geometry AS $$ import spiexceptions from cartodb_services.tools import Logger,LoggerConfig @@ -49,16 +49,16 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) try: - street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, $4, NULL, $5, $6) as point;", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(street_point, [username, orgname, appname, city_name, admin1_name, country_name])[0]['point'] + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] except spiexceptions.ExternalRoutineException as e: import sys logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) - internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5, $6) as point;", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(internal_plan, [username, orgname, appname, city_name, admin1_name, country_name])[0]['point'] + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, appname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig, metrics @@ -74,7 +74,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): try: diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 8390e71..d82f42e 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -15,7 +15,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'code': code} + params = {'username': username, 'orgname': orgname, 'code': code} with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): try: @@ -37,13 +37,13 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code double precision) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, appname, code::text); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code text, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -79,13 +79,13 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, appname TEXT, code double precision, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, appname, code::text, country); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -121,14 +121,14 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code double precision) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, appname, code::text) + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code text, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -164,9 +164,9 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, appname TEXT, code double precision, country text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) RETURNS Geometry AS $$ - SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, appname, code::text, country); + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); $$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 7eb16e3..0f41c5a 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, appname TEXT, ip text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService @@ -15,7 +15,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - params = {'username': username, 'orgname': orgname, 'appname': appname, 'ip': ip} + params = {'username': username, 'orgname': orgname, 'ip': ip} with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): try: diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 3fb7feb..eadd0b8 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -14,27 +14,27 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - params = {'username': username, 'orgname': orgname, 'appname': appname, 'source': source, 'mode': mode, 'range': range, 'options': options} + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- heremaps isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -49,7 +49,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapzen isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -63,7 +63,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapbox isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -77,7 +77,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- tomtom isodistance -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 5a06268..26d73d7 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger @@ -14,27 +14,27 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - params = {'username': username, 'orgname': orgname, 'appname': appname, 'source': source, 'mode': mode, 'range': range, 'options': options} + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapbox_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(tomtom_plan, [username, orgname, appname, source, mode, range, options]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- heremaps isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -49,7 +49,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapzen isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- mapbox isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -75,7 +75,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -- tomtom isochrone -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, appname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] diff --git a/server/extension/test/expected/30_admin0_test.out b/server/extension/test/expected/30_admin0_test.out index 61fb151..bf164ad 100644 --- a/server/extension/test/expected/30_admin0_test.out +++ b/server/extension/test/expected/30_admin0_test.out @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); cdb_geocode_admin0_polygon ---------------------------- @@ -16,7 +16,7 @@ INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( -71.1031880899493 42.3152774590236))',4326) ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); cdb_geocode_admin0_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 diff --git a/server/extension/test/expected/40_admin1_test.out b/server/extension/test/expected/40_admin1_test.out index 80a76ed..000ff55 100644 --- a/server/extension/test/expected/40_admin1_test.out +++ b/server/extension/test/expected/40_admin1_test.out @@ -1,12 +1,12 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); cdb_geocode_admin1_polygon ---------------------------- (1 row) -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); cdb_geocode_admin1_polygon ---------------------------- @@ -22,13 +22,13 @@ INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['c -71.1031880899493 42.3152774590236))',4326) ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); cdb_geocode_admin1_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); cdb_geocode_admin1_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index cab8f50..11fb98b 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -1,20 +1,20 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ------------------------------ @@ -34,42 +34,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E -- Insert dummy data into admin1 decoder table INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.' cdb_geocode_namedplace_point ---------------------------------------------------- diff --git a/server/extension/test/expected/70_ips_test.out b/server/extension/test/expected/70_ips_test.out index f5ea404..39b5886 100644 --- a/server/extension/test/expected/70_ips_test.out +++ b/server/extension/test/expected/70_ips_test.out @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); cdb_geocode_ipaddress_point ----------------------------- @@ -9,7 +9,7 @@ SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_or -- Insert dummy data into ip_address_locations INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); cdb_geocode_ipaddress_point ---------------------------------------------------- 0101000020E61000003333333333334440AE47E17A14AE0D40 diff --git a/server/extension/test/sql/30_admin0_test.sql b/server/extension/test/sql/30_admin0_test.sql index 7a558ea..ac8911b 100644 --- a/server/extension/test/sql/30_admin0_test.sql +++ b/server/extension/test/sql/30_admin0_test.sql @@ -1,6 +1,6 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); -- Insert some dummy synonym INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); @@ -14,7 +14,7 @@ INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'test_appname', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); -- Check for admin0 signatures SELECT exists(SELECT * diff --git a/server/extension/test/sql/40_admin1_test.sql b/server/extension/test/sql/40_admin1_test.sql index d8c963a..547cade 100644 --- a/server/extension/test/sql/40_admin1_test.sql +++ b/server/extension/test/sql/40_admin1_test.sql @@ -1,7 +1,7 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); -- Insert dummy data into country decoder table INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); @@ -15,8 +15,8 @@ INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['c ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'test_appname', 'California', 'United States'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); -- Check for admin1 signatures SELECT exists(SELECT * diff --git a/server/extension/test/sql/50_namedplaces_test.sql b/server/extension/test/sql/50_namedplaces_test.sql index dd19a88..ca601b0 100644 --- a/server/extension/test/sql/50_namedplaces_test.sql +++ b/server/extension/test/sql/50_namedplaces_test.sql @@ -1,8 +1,8 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -- Insert dummy data into points table INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( @@ -21,12 +21,12 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); -- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elx', 'Valencia', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'test_appname', 'Elche', 'valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -- Check for namedplaces signatures SELECT exists(SELECT * diff --git a/server/extension/test/sql/70_ips_test.sql b/server/extension/test/sql/70_ips_test.sql index 79bea61..ba69842 100644 --- a/server/extension/test/sql/70_ips_test.sql +++ b/server/extension/test/sql/70_ips_test.sql @@ -1,12 +1,12 @@ -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); -- Insert dummy data into ip_address_locations INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', 'test_appname', '0.0.0.0'); +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); -- Check for namedplaces signatures (point and polygon) SELECT exists(SELECT * From b4123625f76673a8fe12e613afbe573f648eddd3 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 13:15:19 +0200 Subject: [PATCH 198/265] Fixed server tests --- .../test/expected/20_street_test.out | 2 +- .../test/expected/30_admin0_test.out | 2 +- .../test/expected/40_admin1_test.out | 4 +-- .../test/expected/60_postalcodes_test.out | 34 +++++++++---------- .../extension/test/expected/70_ips_test.out | 2 +- .../test/expected/85_isodistance_test.out | 2 +- .../test/expected/90_isochrone_test.out | 2 +- .../expected/95_route_between_points_test.out | 4 +-- server/extension/test/sql/20_street_test.sql | 2 +- server/extension/test/sql/30_admin0_test.sql | 2 +- server/extension/test/sql/40_admin1_test.sql | 4 +-- .../test/sql/60_postalcodes_test.sql | 26 +++++++------- server/extension/test/sql/70_ips_test.sql | 2 +- .../test/sql/85_isodistance_test.sql | 2 +- .../extension/test/sql/90_isochrone_test.sql | 2 +- .../test/sql/95_route_between_points_test.sql | 4 +-- 16 files changed, 48 insertions(+), 48 deletions(-) diff --git a/server/extension/test/expected/20_street_test.out b/server/extension/test/expected/20_street_test.out index 3f02e6e..8fbce7a 100644 --- a/server/extension/test/expected/20_street_test.out +++ b/server/extension/test/expected/20_street_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/30_admin0_test.out b/server/extension/test/expected/30_admin0_test.out index bf164ad..55c7639 100644 --- a/server/extension/test/expected/30_admin0_test.out +++ b/server/extension/test/expected/30_admin0_test.out @@ -28,7 +28,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); exists -------- t diff --git a/server/extension/test/expected/40_admin1_test.out b/server/extension/test/expected/40_admin1_test.out index 000ff55..4894b0c 100644 --- a/server/extension/test/expected/40_admin1_test.out +++ b/server/extension/test/expected/40_admin1_test.out @@ -40,7 +40,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); exists -------- t @@ -51,7 +51,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t diff --git a/server/extension/test/expected/60_postalcodes_test.out b/server/extension/test/expected/60_postalcodes_test.out index f671d79..bc7ac56 100644 --- a/server/extension/test/expected/60_postalcodes_test.out +++ b/server/extension/test/expected/60_postalcodes_test.out @@ -42,25 +42,25 @@ INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( 3 ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); cdb_geocode_postalcode_point ---------------------------------------------------- 0101000020E61000000000000000E040408036B47414764840 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); cdb_geocode_postalcode_point ---------------------------------------------------- 0101000020E61000000000000000E040408036B47414764840 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); cdb_geocode_postalcode_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 (1 row) -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); cdb_geocode_postalcode_polygon -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 @@ -78,7 +78,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); exists -------- t @@ -89,7 +89,18 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); exists -------- t @@ -106,17 +117,6 @@ SELECT exists(SELECT * t (1 row) -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); - exists --------- - t -(1 row) - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/expected/70_ips_test.out b/server/extension/test/expected/70_ips_test.out index 39b5886..48012f7 100644 --- a/server/extension/test/expected/70_ips_test.out +++ b/server/extension/test/expected/70_ips_test.out @@ -21,7 +21,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); exists -------- t diff --git a/server/extension/test/expected/85_isodistance_test.out b/server/extension/test/expected/85_isodistance_test.out index 2ecbb88..5705284 100644 --- a/server/extension/test/expected/85_isodistance_test.out +++ b/server/extension/test/expected/85_isodistance_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); exists -------- t diff --git a/server/extension/test/expected/90_isochrone_test.out b/server/extension/test/expected/90_isochrone_test.out index 193b59f..7ee0f04 100644 --- a/server/extension/test/expected/90_isochrone_test.out +++ b/server/extension/test/expected/90_isochrone_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); exists -------- t diff --git a/server/extension/test/expected/95_route_between_points_test.out b/server/extension/test/expected/95_route_between_points_test.out index cfc8585..176059f 100644 --- a/server/extension/test/expected/95_route_between_points_test.out +++ b/server/extension/test/expected/95_route_between_points_test.out @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_point_to_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, geometry, text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); exists -------- t @@ -16,7 +16,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_with_waypoints' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry[], text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry[], text, text[], text'); exists -------- t diff --git a/server/extension/test/sql/20_street_test.sql b/server/extension/test/sql/20_street_test.sql index 1843679..86e0368 100644 --- a/server/extension/test/sql/20_street_test.sql +++ b/server/extension/test/sql/20_street_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); diff --git a/server/extension/test/sql/30_admin0_test.sql b/server/extension/test/sql/30_admin0_test.sql index ac8911b..3249c60 100644 --- a/server/extension/test/sql/30_admin0_test.sql +++ b/server/extension/test/sql/30_admin0_test.sql @@ -22,7 +22,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/40_admin1_test.sql b/server/extension/test/sql/40_admin1_test.sql index 547cade..7b3748c 100644 --- a/server/extension/test/sql/40_admin1_test.sql +++ b/server/extension/test/sql/40_admin1_test.sql @@ -24,14 +24,14 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/60_postalcodes_test.sql b/server/extension/test/sql/60_postalcodes_test.sql index eea2c8f..180def8 100644 --- a/server/extension/test/sql/60_postalcodes_test.sql +++ b/server/extension/test/sql/60_postalcodes_test.sql @@ -44,13 +44,13 @@ INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( ); -- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', 'test_app', '03204', 'spain'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); -- Clean dbs DELETE FROM global_postal_code_points; @@ -65,14 +65,21 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); SELECT exists(SELECT * FROM pg_proc p @@ -81,13 +88,6 @@ SELECT exists(SELECT * AND proname = 'cdb_geocode_postalcode_polygon' AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/70_ips_test.sql b/server/extension/test/sql/70_ips_test.sql index ba69842..6111bd0 100644 --- a/server/extension/test/sql/70_ips_test.sql +++ b/server/extension/test/sql/70_ips_test.sql @@ -14,7 +14,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/85_isodistance_test.sql b/server/extension/test/sql/85_isodistance_test.sql index 4f5c37f..f66b68a 100644 --- a/server/extension/test/sql/85_isodistance_test.sql +++ b/server/extension/test/sql/85_isodistance_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/sql/90_isochrone_test.sql b/server/extension/test/sql/90_isochrone_test.sql index 60819bc..113fb28 100644 --- a/server/extension/test/sql/90_isochrone_test.sql +++ b/server/extension/test/sql/90_isochrone_test.sql @@ -4,4 +4,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, text, integer[], text[]'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/sql/95_route_between_points_test.sql b/server/extension/test/sql/95_route_between_points_test.sql index 21fe257..4acc15d 100644 --- a/server/extension/test/sql/95_route_between_points_test.sql +++ b/server/extension/test/sql/95_route_between_points_test.sql @@ -4,7 +4,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_point_to_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry, geometry, text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); -- Check for routing waypoint route signatures SELECT exists(SELECT * @@ -12,4 +12,4 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_route_with_waypoints' - AND oidvectortypes(p.proargtypes) = 'text, text, text, geometry[], text, text[], text'); + AND oidvectortypes(p.proargtypes) = 'text, text, geometry[], text, text[], text'); From 01f145ab69c4abd4b140d15f1ec3364fdb6e9232 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 13:40:15 +0200 Subject: [PATCH 199/265] Fixed client tests --- client/test/expected/10_admin0_test.out | 6 ++--- client/test/expected/20_admin1_test.out | 8 +++--- ..._exception_safe_private_functions_test.out | 4 +-- client/test/expected/30_namedplaces_test.out | 10 +++---- client/test/expected/40_postalcodes_test.out | 8 +++--- client/test/expected/50_ipaddresses_test.out | 6 ++--- client/test/expected/60_street_test.out | 24 ++++++++--------- .../expected/80_route_point_to_point_test.out | 16 ++++++------ client/test/expected/999_permissions_test.out | 26 +++++++++---------- client/test/sql/10_admin0_test.sql | 4 +-- client/test/sql/20_admin1_test.sql | 4 +-- ..._exception_safe_private_functions_test.sql | 4 +-- client/test/sql/30_namedplaces_test.sql | 4 +-- client/test/sql/40_postalcodes_test.sql | 4 +-- client/test/sql/50_ipaddresses_test.sql | 4 +-- client/test/sql/60_street_test.sql | 4 +-- .../test/sql/80_route_point_to_point_test.sql | 4 +-- client/test/sql/999_permissions_test.sql | 4 +-- 18 files changed, 72 insertions(+), 72 deletions(-) diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 190fef3..3df63e9 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -14,7 +14,7 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_admin0_polygon('Spain'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -23,14 +23,14 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "pe SELECT cdb_geocode_admin0_polygon('Spain'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , "testing_app", Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) cdb_geocode_admin0_polygon ---------------------------- diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index 590fb8f..62ee741 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -23,7 +23,7 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_admin1_polygon('California', 'United States'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -34,21 +34,21 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_admin1_polygon('California', 'United States'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California, United States) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) cdb_geocode_admin1_polygon ---------------------------- diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index 65d9df7..b968323 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -38,7 +38,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 ERROR: Routing is not allowed -- No permissions granted but conf created SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": []}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); cdb_conf_setconf ------------------ @@ -53,7 +53,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 ERROR: Routing is not allowed -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing", "isolines"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index 1936f54..acbb6b7 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -32,7 +32,7 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -45,28 +45,28 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) cdb_geocode_namedplace_point ------------------------------ diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index 5af322f..b3a41e7 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -23,7 +23,7 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_postalcode_point('03204', 'Spain'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -34,21 +34,21 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_postalcode_point('03204', 'Spain'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , "testing_app", 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) cdb_geocode_postalcode_polygon -------------------------------- (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , "testing_app", 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) cdb_geocode_postalcode_point ------------------------------ diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index fac1696..72e1bf2 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -14,7 +14,7 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_ipaddress_point('8.8.8.8'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -23,14 +23,14 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "pe SELECT cdb_geocode_ipaddress_point('8.8.8.8'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , "testing_app", 8.8.8.8) +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) cdb_geocode_ipaddress_point ----------------------------- diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index 2289058..c64d4f6 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -32,7 +32,7 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -59,77 +59,77 @@ ERROR: Geocoding is not allowed SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, state, country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", One street, city, , country) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) cdb_geocode_street_point -------------------------- diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index 2984bbf..9fea2a6 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -37,7 +37,7 @@ ERROR: Routing is not allowed SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); ERROR: Routing is not allowed -- Grant other permissions but routing -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "isolines"]}'); cdb_conf_setconf ------------------ @@ -56,49 +56,49 @@ ERROR: Routing is not allowed SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); ERROR: Routing is not allowed -- Grant routing permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing"]}'); cdb_conf_setconf ------------------ (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) cdb_route_point_to_point -------------------------- (,5.33,100) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {mode_type=shortest}, kilometers) cdb_route_with_waypoints -------------------------- (,2.22,500) (1 row) SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , "testing_app", {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) +NOTICE: cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_with_waypoints invoked with params (test_user, , {0101000000D53E1D8F19F455C0185B087250F24440:0101000000650113B875F355C05665DF15C1EF4440:0101000000465F419AB1F255C0D8B628B341EE4440}, car, {}, miles) cdb_route_with_waypoints -------------------------- (,2.22,500) diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index c359e60..21642ab 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -53,7 +53,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 ERROR: Routing is not allowed -- Grant other permissions but geocoding and routing SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); cdb_conf_setconf ------------------ @@ -84,7 +84,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 ERROR: Routing is not allowed -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing"]}'); cdb_conf_setconf ------------------ @@ -92,77 +92,77 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "pe SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , "testing_app", Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) cdb_geocode_admin0_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , "testing_app", California, United States) +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) cdb_geocode_admin1_polygon ---------------------------- (1 row) SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , "testing_app", Elx, Valencia, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) cdb_geocode_namedplace_point ------------------------------ (1 row) SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , "testing_app", 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) cdb_geocode_postalcode_polygon -------------------------------- (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , "testing_app", 03204, Spain) +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) cdb_geocode_postalcode_point ------------------------------ (1 row) SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , "testing_app", 8.8.8.8) +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) cdb_geocode_ipaddress_point ----------------------------- (1 row) SELECT cdb_geocode_street_point('one street, 1'); -NOTICE: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , "testing_app", one street, 1, , , ) +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , one street, 1, , , ) cdb_geocode_street_point -------------------------- (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -NOTICE: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , "testing_app", 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) cdb_route_point_to_point -------------------------- (,5.33,100) diff --git a/client/test/sql/10_admin0_test.sql b/client/test/sql/10_admin0_test.sql index f5e1bd0..4dfc2a6 100644 --- a/client/test/sql/10_admin0_test.sql +++ b/client/test/sql/10_admin0_test.sql @@ -17,11 +17,11 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_admin0_polygon('Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_admin0_polygon('Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_admin0_polygon('Spain'); -- Remove permissions diff --git a/client/test/sql/20_admin1_test.sql b/client/test/sql/20_admin1_test.sql index fd2250d..2f8e0f4 100644 --- a/client/test/sql/20_admin1_test.sql +++ b/client/test/sql/20_admin1_test.sql @@ -26,12 +26,12 @@ SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index 45dbfca..b2d431f 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -42,7 +42,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 -- No permissions granted but conf created SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": []}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{permissions": []}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); @@ -50,7 +50,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing", "isolines"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); diff --git a/client/test/sql/30_namedplaces_test.sql b/client/test/sql/30_namedplaces_test.sql index e410a9c..f973b94 100644 --- a/client/test/sql/30_namedplaces_test.sql +++ b/client/test/sql/30_namedplaces_test.sql @@ -35,13 +35,13 @@ SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_namedplace_point('Elx'); SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_namedplace_point('Elx'); SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); diff --git a/client/test/sql/40_postalcodes_test.sql b/client/test/sql/40_postalcodes_test.sql index ffb2d1b..707ebfa 100644 --- a/client/test/sql/40_postalcodes_test.sql +++ b/client/test/sql/40_postalcodes_test.sql @@ -26,12 +26,12 @@ SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); diff --git a/client/test/sql/50_ipaddresses_test.sql b/client/test/sql/50_ipaddresses_test.sql index 901e21b..36c8b96 100644 --- a/client/test/sql/50_ipaddresses_test.sql +++ b/client/test/sql/50_ipaddresses_test.sql @@ -18,11 +18,11 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Remove permissions diff --git a/client/test/sql/60_street_test.sql b/client/test/sql/60_street_test.sql index 17a7a61..e47c124 100644 --- a/client/test/sql/60_street_test.sql +++ b/client/test/sql/60_street_test.sql @@ -27,7 +27,7 @@ SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_street_point('One street, 1'); SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); @@ -40,7 +40,7 @@ SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); SELECT cdb_geocode_street_point('One street, 1'); SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); diff --git a/client/test/sql/80_route_point_to_point_test.sql b/client/test/sql/80_route_point_to_point_test.sql index fcc04aa..bff1c6d 100644 --- a/client/test/sql/80_route_point_to_point_test.sql +++ b/client/test/sql/80_route_point_to_point_test.sql @@ -39,7 +39,7 @@ SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POI SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -- Grant other permissions but routing -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "isolines"]}'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); @@ -48,7 +48,7 @@ SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POI SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -- Grant routing permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing"]}'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); diff --git a/client/test/sql/999_permissions_test.sql b/client/test/sql/999_permissions_test.sql index 3adb266..bac1254 100644 --- a/client/test/sql/999_permissions_test.sql +++ b/client/test/sql/999_permissions_test.sql @@ -35,7 +35,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 -- Grant other permissions but geocoding and routing SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); SELECT cdb_geocode_admin1_polygon('California'); @@ -51,7 +51,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding", "routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); SELECT cdb_geocode_admin1_polygon('California'); From f5ad0290b4c91b885e504eec902f0bad91d9864c Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 13:43:04 +0200 Subject: [PATCH 200/265] Fixed functions --- .../templates/25_exception_safe_private_functions.erb | 2 +- client/sql/20_table_augmentation.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index 0c5cdb2..4a12b05 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -19,7 +19,7 @@ BEGIN <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; END IF; - <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, a text, p json);<% end %><% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %><% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/sql/20_table_augmentation.sql b/client/sql/20_table_augmentation.sql index a0a0c2f..abcc29e 100644 --- a/client/sql/20_table_augmentation.sql +++ b/client/sql/20_table_augmentation.sql @@ -18,7 +18,7 @@ BEGIN SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument'; @@ -62,7 +62,7 @@ BEGIN SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, a text, p json); + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument'; From 696c24e43209408774a1ebbbadfe8776765d5213 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 13:47:35 +0200 Subject: [PATCH 201/265] Fixed test --- client/test/sql/25_exception_safe_private_functions_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index b2d431f..602817e 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -42,7 +42,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 -- No permissions granted but conf created SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{permissions": []}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); From 3d42281d2a2fe2e34e90237b9af8d4f974bd4813 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 13:50:52 +0200 Subject: [PATCH 202/265] Fixed test --- .../expected/25_exception_safe_private_functions_test.out | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index b968323..da25c75 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -61,20 +61,20 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "rou SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); -WARNING: cdb_dataservices_client._cdb_geocode_street_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. _cdb_geocode_street_point_exception_safe ------------------------------------------ (1 row) SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); -WARNING: cdb_dataservices_client._cdb_isodistance(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. center | data_range | the_geom --------+------------+---------- (0 rows) SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -WARNING: cdb_dataservices_client._cdb_route_point_to_point(8): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. +WARNING: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever. shape | length | duration -------+--------+---------- | | From 49bcf31832d2b29fb150fdc5deedd06062bd36e6 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 14:26:53 +0200 Subject: [PATCH 203/265] Removed appname parameter --- server/extension/sql/85_isodistance.sql | 8 ++++---- server/extension/sql/90_isochrone.sql | 8 ++++---- .../test/expected/60_postalcodes_test.out | 2 +- .../extension/test/sql/50_namedplaces_test.sql | 16 ++++++++-------- .../extension/test/sql/60_postalcodes_test.sql | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index eadd0b8..fbe192e 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -18,16 +18,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ with metrics('cdb_isodistance', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 26d73d7..731a27e 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -18,16 +18,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ with metrics('cdb_isochrone', user_isolines_config, logger, params): if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) else: raise Exception('Requested isolines provider is not available') diff --git a/server/extension/test/expected/60_postalcodes_test.out b/server/extension/test/expected/60_postalcodes_test.out index bc7ac56..b2a5c91 100644 --- a/server/extension/test/expected/60_postalcodes_test.out +++ b/server/extension/test/expected/60_postalcodes_test.out @@ -6,7 +6,7 @@ DELETE FROM available_services; DELETE FROM admin0_synonyms; -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); cdb_geocode_postalcode_point ------------------------------ diff --git a/server/extension/test/sql/50_namedplaces_test.sql b/server/extension/test/sql/50_namedplaces_test.sql index ca601b0..0fde77f 100644 --- a/server/extension/test/sql/50_namedplaces_test.sql +++ b/server/extension/test/sql/50_namedplaces_test.sql @@ -29,6 +29,13 @@ SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_o SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); -- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) @@ -41,14 +48,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); SELECT exists(SELECT * FROM pg_proc p diff --git a/server/extension/test/sql/60_postalcodes_test.sql b/server/extension/test/sql/60_postalcodes_test.sql index 180def8..55d1c85 100644 --- a/server/extension/test/sql/60_postalcodes_test.sql +++ b/server/extension/test/sql/60_postalcodes_test.sql @@ -7,7 +7,7 @@ DELETE FROM admin0_synonyms; -- Check that the public function is callable, even with no data -- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', 'test_app', '03204'); +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); -- Insert dummy data into ip_address_locations INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( From a33dd4315cc32d2215ff7df644b8f54a7b8dfb26 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 14:31:08 +0200 Subject: [PATCH 204/265] Fixe --- .../test/expected/50_namedplaces_test.out | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/extension/test/expected/50_namedplaces_test.out b/server/extension/test/expected/50_namedplaces_test.out index 11fb98b..2e41285 100644 --- a/server/extension/test/expected/50_namedplaces_test.out +++ b/server/extension/test/expected/50_namedplaces_test.out @@ -77,6 +77,17 @@ WARNING: Error geocoding namedplace using geocode street point, falling back to (1 row) -- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) @@ -93,18 +104,7 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); exists -------- t From f4cb9ba8124c792cecbb41e743ab27f90eec1dee Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 14:33:54 +0200 Subject: [PATCH 205/265] Fixe --- client/test/expected/21_bulk_geocoding_functions_test.out | 4 ++-- client/test/sql/21_bulk_geocoding_functions_test.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index aa1b68b..da5d74a 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -17,7 +17,7 @@ ERROR: Geocoding is not allowed SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Geocoding is not allowed -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -30,7 +30,7 @@ ERROR: Geocoding is not allowed SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Geocoding is not allowed -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index 649d8ce..01838b5 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -19,14 +19,14 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"application": "testing_app", "permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -- Test quota check by mocking quota 0 From 08daa6a6a2e450191949ab6f234f469b6b9fdd51 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 16:45:31 +0200 Subject: [PATCH 206/265] Changed error message --- client/renderer/interface.yaml | 40 ++++++++--------- .../templates/20_public_functions.erb | 6 +-- client/sql/21_bulk_geocoding_functions.sql | 2 +- client/test/expected/10_admin0_test.out | 4 +- client/test/expected/20_admin1_test.out | 8 ++-- .../21_bulk_geocoding_functions_test.out | 8 ++-- ..._exception_safe_private_functions_test.out | 8 ++-- client/test/expected/30_namedplaces_test.out | 12 ++--- client/test/expected/40_postalcodes_test.out | 8 ++-- client/test/expected/50_ipaddresses_test.out | 4 +- client/test/expected/60_street_test.out | 40 ++++++++--------- .../expected/80_route_point_to_point_test.out | 24 +++++----- client/test/expected/999_permissions_test.out | 44 +++++++++---------- 13 files changed, 104 insertions(+), 104 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 97ad5cc..95d7e88 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -3,7 +3,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: country_name, type: text } @@ -11,7 +11,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: admin1_name, type: text } @@ -19,7 +19,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: admin1_name, type: text } - { name: country_name, type: text } @@ -28,7 +28,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: city_name, type: text} @@ -36,7 +36,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: city_name, type: text} - { name: country_name, type: text} @@ -45,7 +45,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: city_name, type: text} - { name: admin1_name, type: text} @@ -55,7 +55,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: postal_code, type: text} - { name: country_name, type: text} @@ -64,7 +64,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: postal_code, type: double precision} - { name: country_name, type: text} @@ -73,7 +73,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: postal_code, type: text} - { name: country_name, type: text} @@ -82,7 +82,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: postal_code, type: double precision} - { name: country_name, type: text} @@ -91,7 +91,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: ip_address, type: text} @@ -99,7 +99,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -112,7 +112,7 @@ multi_field: true requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searches, type: jsonb } # Array of JSON objects with id, address, city, state and country fields @@ -120,7 +120,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -131,7 +131,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -142,7 +142,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -153,7 +153,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -164,7 +164,7 @@ return_type: Geometry requires_permission: true permission_name: geocoding - permission_error: Geocoding is not allowed + permission_error: Geocoding permission denied params: - { name: searchtext, type: text} - { name: city, type: text, default: 'NULL'} @@ -280,7 +280,7 @@ multi_field: true requires_permission: true permission_name: routing - permission_error: Routing is not allowed + permission_error: Routing permission denied params: - { name: origin, type: "geometry(Point, 4326)" } - { name: destination, type: "geometry(Point, 4326)" } @@ -293,7 +293,7 @@ multi_field: true requires_permission: true permission_name: routing - permission_error: Routing is not allowed + permission_error: Routing permission denied params: - { name: waypoints, type: "geometry(Point, 4326)[]" } - { name: mode, type: text } diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 3950037..07953aa 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -16,9 +16,9 @@ BEGIN END IF; <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN - RAISE EXCEPTION '<%= permission_error %>'; - END IF; - <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %><% end %> + RAISE EXCEPTION '<%= permission_error %>' USING ERRCODE = 'CDB403'; + END IF;<% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 5fd9284..0276836 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -22,7 +22,7 @@ BEGIN END IF; SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN - RAISE EXCEPTION 'Geocoding is not allowed'; + RAISE EXCEPTION 'Geocoding permission denied'; END IF; -- JSON value stored "" is taken as literal diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 3df63e9..9046dce 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -12,7 +12,7 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_admin0_polygon('Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -21,7 +21,7 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_admin0_polygon('Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index 62ee741..4c1839e 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -19,9 +19,9 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_admin1_polygon('California'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -30,9 +30,9 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_admin1_polygon('California'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index da5d74a..16c370d 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -12,10 +12,10 @@ $$ LANGUAGE SQL; -- No permissions granted -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -25,10 +25,10 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index da25c75..444c35f 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -31,11 +31,11 @@ $$ LANGUAGE 'plpgsql'; -- No permissions granted SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); ERROR: Isolines are not allowed SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- No permissions granted but conf created SET ROLE postgres; SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); @@ -46,11 +46,11 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); ERROR: Isolines are not allowed SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- Grant geocoding permissions SET ROLE postgres; SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing", "isolines"]}'); diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index acbb6b7..095d4b3 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -26,11 +26,11 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_namedplace_point('Elx'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -39,11 +39,11 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_namedplace_point('Elx'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index b3a41e7..e66f912 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -19,9 +19,9 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -30,9 +30,9 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index 72e1bf2..36acf41 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -12,7 +12,7 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -21,7 +21,7 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index c64d4f6..ce4df0b 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -12,25 +12,25 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_geocode_street_point('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant other permissions but geocoding SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); cdb_conf_setconf @@ -39,25 +39,25 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli (1 row) SELECT cdb_geocode_street_point('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied -- Grant geocoding permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); cdb_conf_setconf diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index 9fea2a6..762de95 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -25,17 +25,17 @@ $$ LANGUAGE 'plpgsql'; -- -- Exercise the public and the proxied function -- No permissions granted SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- Grant other permissions but routing SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "isolines"]}'); cdb_conf_setconf @@ -44,17 +44,17 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "iso (1 row) SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]); -ERROR: Routing is not allowed +ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- Grant routing permissions SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing"]}'); cdb_conf_setconf diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index 21642ab..945f1dc 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -30,27 +30,27 @@ ERROR: permission denied for function _cdb_route_point_to_point -- -- it is public, it shall work -- No permissions granted SELECT cdb_geocode_admin0_polygon('Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('one street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- Grant other permissions but geocoding and routing SET ROLE postgres; SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); @@ -61,27 +61,27 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('one street, 1'); -ERROR: Geocoding is not allowed +ERROR: Geocoding permission denied SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); -ERROR: Routing is not allowed +ERROR: Routing permission denied -- Grant geocoding permissions SET ROLE postgres; SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing"]}'); From 0bdba673423eebb156bc34ba10286b20b3a14839 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 16:54:25 +0200 Subject: [PATCH 207/265] Using ERRCODE --- client/renderer/templates/20_public_functions.erb | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 07953aa..a5c4fcf 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -16,7 +16,7 @@ BEGIN END IF; <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN - RAISE EXCEPTION '<%= permission_error %>' USING ERRCODE = 'CDB403'; + RAISE EXCEPTION '<%= permission_error %>' USING ERRCODE = '01007'; END IF;<% end %> <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %> -- JSON value stored "" is taken as literal diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 0276836..d4d7c5d 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -22,7 +22,7 @@ BEGIN END IF; SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN - RAISE EXCEPTION 'Geocoding permission denied'; + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; END IF; -- JSON value stored "" is taken as literal From d5864ccada2a261cf59eef3a809b999fd9a6dda4 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Mon, 10 Sep 2018 17:06:37 +0200 Subject: [PATCH 208/265] Using ERRCODE --- client/renderer/interface.yaml | 16 ++++++++-------- .../25_exception_safe_private_functions_test.out | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 95d7e88..06d57e6 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -177,7 +177,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -190,7 +190,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -203,7 +203,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -216,7 +216,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -229,7 +229,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -242,7 +242,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -255,7 +255,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } @@ -268,7 +268,7 @@ multi_field: true requires_permission: true permission_name: isolines - permission_error: Isolines are not allowed + permission_error: Isolines permission denied params: - { name: source, type: "geometry(Geometry, 4326)" } - { name: mode, type: text } diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index 444c35f..c5bcd1f 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -33,7 +33,7 @@ SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); ERROR: Geocoding permission denied SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); -ERROR: Isolines are not allowed +ERROR: Isolines permission denied SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); ERROR: Routing permission denied -- No permissions granted but conf created @@ -48,7 +48,7 @@ SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); ERROR: Geocoding permission denied SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); -ERROR: Isolines are not allowed +ERROR: Isolines permission denied SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); ERROR: Routing permission denied -- Grant geocoding permissions From 4d3e9f2b32d0d88a41c94023e9f9e07b0f9952c9 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 10:14:52 +0200 Subject: [PATCH 209/265] Getting the username from cdb_conf --- client/sql/15_config_management.sql | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index d1e92f5..7e03c3f 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -25,16 +25,11 @@ BEGIN IF is_organization IS NULL THEN RAISE EXCEPTION 'User must have user configuration in the config table'; ELSIF is_organization = TRUE THEN - SELECT nspname - FROM pg_namespace s - LEFT JOIN pg_roles r ON s.nspowner = r.oid - WHERE r.rolname = session_user INTO username; SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; ELSE - SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; organization_name = NULL; END IF; - result.username = username; + result.username = apikey_config->'username'; result.organization_name = organization_name; result.apikey_permissions = apikey_config->'permissions'; RETURN result; From e82fb49028f81db39cb63d4a080da8f8151585e3 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 10:57:32 +0200 Subject: [PATCH 210/265] organization default to null --- client/sql/15_config_management.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index 7e03c3f..73f34ec 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -17,7 +17,7 @@ DECLARE apikey_config json; is_organization boolean; username text; - organization_name text; + organization_name text DEFAULT NULL; BEGIN SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; @@ -26,8 +26,6 @@ BEGIN RAISE EXCEPTION 'User must have user configuration in the config table'; ELSIF is_organization = TRUE THEN SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; - ELSE - organization_name = NULL; END IF; result.username = apikey_config->'username'; result.organization_name = organization_name; From b6d37c677309094122653b4a6da6accf47c6f545 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 11:07:59 +0200 Subject: [PATCH 211/265] Managing public api key permissions --- client/renderer/templates/20_public_functions.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index a5c4fcf..309f1cb 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -11,9 +11,9 @@ DECLARE <%= user_org_declaration %> <% unless superuser_function? -%>apikey_permissions json;<% end %> BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + <% unless requires_permission %>IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; - END IF; + END IF;<% end %> <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>' USING ERRCODE = '01007'; From c5b6c270452f83979b807909ec4f56d4e5fd5f8f Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 11:19:12 +0200 Subject: [PATCH 212/265] Fixed JSON return value --- client/sql/15_config_management.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index 73f34ec..58cd1ea 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -27,7 +27,7 @@ BEGIN ELSIF is_organization = TRUE THEN SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; END IF; - result.username = apikey_config->'username'; + result.username = apikey_config->>'username'; result.organization_name = organization_name; result.apikey_permissions = apikey_config->'permissions'; RETURN result; From 2a6e63b6182e7561742f688799afe5d9aea87fbc Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 12:02:24 +0200 Subject: [PATCH 213/265] Fixed tests --- client/test/expected/10_admin0_test.out | 4 ++-- client/test/expected/20_admin1_test.out | 4 ++-- .../expected/21_bulk_geocoding_functions_test.out | 4 ++-- .../25_exception_safe_private_functions_test.out | 4 ++-- client/test/expected/30_namedplaces_test.out | 4 ++-- client/test/expected/40_postalcodes_test.out | 4 ++-- client/test/expected/50_ipaddresses_test.out | 4 ++-- client/test/expected/60_street_test.out | 4 ++-- .../test/expected/80_route_point_to_point_test.out | 4 ++-- client/test/expected/90_data_observatory_test.out | 12 ++++++++++++ .../expected/95_data_observatory_tables_test.out | 12 ++++++++++++ client/test/expected/999_permissions_test.out | 4 ++-- client/test/sql/10_admin0_test.sql | 4 ++-- client/test/sql/20_admin1_test.sql | 4 ++-- client/test/sql/21_bulk_geocoding_functions_test.sql | 4 ++-- .../sql/25_exception_safe_private_functions_test.sql | 4 ++-- client/test/sql/30_namedplaces_test.sql | 4 ++-- client/test/sql/40_postalcodes_test.sql | 4 ++-- client/test/sql/50_ipaddresses_test.sql | 4 ++-- client/test/sql/60_street_test.sql | 4 ++-- client/test/sql/80_route_point_to_point_test.sql | 4 ++-- client/test/sql/90_data_observatory_test.sql | 3 +++ client/test/sql/95_data_observatory_tables_test.sql | 6 +++++- client/test/sql/999_permissions_test.sql | 4 ++-- 24 files changed, 72 insertions(+), 41 deletions(-) diff --git a/client/test/expected/10_admin0_test.out b/client/test/expected/10_admin0_test.out index 9046dce..c36e98e 100644 --- a/client/test/expected/10_admin0_test.out +++ b/client/test/expected/10_admin0_test.out @@ -14,7 +14,7 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_admin0_polygon('Spain'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -23,7 +23,7 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli SELECT cdb_geocode_admin0_polygon('Spain'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/20_admin1_test.out b/client/test/expected/20_admin1_test.out index 4c1839e..006695a 100644 --- a/client/test/expected/20_admin1_test.out +++ b/client/test/expected/20_admin1_test.out @@ -23,7 +23,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -34,7 +34,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_admin1_polygon('California', 'United States'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index 16c370d..beebcab 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -17,7 +17,7 @@ ERROR: Geocoding permission denied SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -30,7 +30,7 @@ ERROR: Geocoding permission denied SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/25_exception_safe_private_functions_test.out b/client/test/expected/25_exception_safe_private_functions_test.out index c5bcd1f..563aa50 100644 --- a/client/test/expected/25_exception_safe_private_functions_test.out +++ b/client/test/expected/25_exception_safe_private_functions_test.out @@ -38,7 +38,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 ERROR: Routing permission denied -- No permissions granted but conf created SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": []}'); cdb_conf_setconf ------------------ @@ -53,7 +53,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 ERROR: Routing permission denied -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing", "isolines"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/30_namedplaces_test.out b/client/test/expected/30_namedplaces_test.out index 095d4b3..bc26eb4 100644 --- a/client/test/expected/30_namedplaces_test.out +++ b/client/test/expected/30_namedplaces_test.out @@ -32,7 +32,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -45,7 +45,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/40_postalcodes_test.out b/client/test/expected/40_postalcodes_test.out index e66f912..afca43a 100644 --- a/client/test/expected/40_postalcodes_test.out +++ b/client/test/expected/40_postalcodes_test.out @@ -23,7 +23,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -34,7 +34,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_postalcode_point('03204', 'Spain'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/50_ipaddresses_test.out b/client/test/expected/50_ipaddresses_test.out index 36acf41..6ebcbbd 100644 --- a/client/test/expected/50_ipaddresses_test.out +++ b/client/test/expected/50_ipaddresses_test.out @@ -14,7 +14,7 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_ipaddress_point('8.8.8.8'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -23,7 +23,7 @@ SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isoli SELECT cdb_geocode_ipaddress_point('8.8.8.8'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/60_street_test.out b/client/test/expected/60_street_test.out index ce4df0b..b3d9604 100644 --- a/client/test/expected/60_street_test.out +++ b/client/test/expected/60_street_test.out @@ -32,7 +32,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); ERROR: Geocoding permission denied -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); cdb_conf_setconf ------------------ @@ -59,7 +59,7 @@ ERROR: Geocoding permission denied SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); ERROR: Geocoding permission denied -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/80_route_point_to_point_test.out b/client/test/expected/80_route_point_to_point_test.out index 762de95..c5bc94f 100644 --- a/client/test/expected/80_route_point_to_point_test.out +++ b/client/test/expected/80_route_point_to_point_test.out @@ -37,7 +37,7 @@ ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); ERROR: Routing permission denied -- Grant other permissions but routing -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "isolines"]}'); cdb_conf_setconf ------------------ @@ -56,7 +56,7 @@ ERROR: Routing permission denied SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); ERROR: Routing permission denied -- Grant routing permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/expected/90_data_observatory_test.out b/client/test/expected/90_data_observatory_test.out index d461dde..05b429d 100644 --- a/client/test/expected/90_data_observatory_test.out +++ b/client/test/expected/90_data_observatory_test.out @@ -202,6 +202,12 @@ BEGIN RETURN QUERY SELECT true AS valid, ARRAY[]::TEXT[] AS errors; END; $$ LANGUAGE 'plpgsql'; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); + cdb_conf_setconf +------------------ + +(1 row) + -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group) @@ -366,3 +372,9 @@ NOTICE: cdb_dataservices_client._obs_metadatavalidation(6): [contrib_regression (t,{}) (1 row) +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/95_data_observatory_tables_test.out b/client/test/expected/95_data_observatory_tables_test.out index aaf97e7..594305e 100644 --- a/client/test/expected/95_data_observatory_tables_test.out +++ b/client/test/expected/95_data_observatory_tables_test.out @@ -27,6 +27,12 @@ BEGIN RETURN true; END; $$ LANGUAGE 'plpgsql'; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); + cdb_conf_setconf +------------------ + +(1 row) + -- Create a sample user table CREATE TABLE user_table (cartodb_id int, the_geom geometry); INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440'); @@ -63,3 +69,9 @@ SELECT * FROM my_table_dst; -- Clean tables DROP TABLE my_table_dst; +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); + cdb_conf_removeconf +--------------------- + +(1 row) + diff --git a/client/test/expected/999_permissions_test.out b/client/test/expected/999_permissions_test.out index 945f1dc..7f304b1 100644 --- a/client/test/expected/999_permissions_test.out +++ b/client/test/expected/999_permissions_test.out @@ -53,7 +53,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 ERROR: Routing permission denied -- Grant other permissions but geocoding and routing SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["isolines"]}'); cdb_conf_setconf ------------------ @@ -84,7 +84,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 ERROR: Routing permission denied -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing"]}'); cdb_conf_setconf ------------------ diff --git a/client/test/sql/10_admin0_test.sql b/client/test/sql/10_admin0_test.sql index 4dfc2a6..69ee0be 100644 --- a/client/test/sql/10_admin0_test.sql +++ b/client/test/sql/10_admin0_test.sql @@ -17,11 +17,11 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_admin0_polygon('Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_admin0_polygon('Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_admin0_polygon('Spain'); -- Remove permissions diff --git a/client/test/sql/20_admin1_test.sql b/client/test/sql/20_admin1_test.sql index 2f8e0f4..829546a 100644 --- a/client/test/sql/20_admin1_test.sql +++ b/client/test/sql/20_admin1_test.sql @@ -26,12 +26,12 @@ SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_admin1_polygon('California'); SELECT cdb_geocode_admin1_polygon('California', 'United States'); diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index 01838b5..5b899c8 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -19,14 +19,14 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); -- Test quota check by mocking quota 0 diff --git a/client/test/sql/25_exception_safe_private_functions_test.sql b/client/test/sql/25_exception_safe_private_functions_test.sql index 602817e..9b4f9fd 100644 --- a/client/test/sql/25_exception_safe_private_functions_test.sql +++ b/client/test/sql/25_exception_safe_private_functions_test.sql @@ -42,7 +42,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 -- No permissions granted but conf created SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": []}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": []}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); @@ -50,7 +50,7 @@ SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.417 -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing", "isolines"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT _cdb_geocode_street_point_exception_safe('One street, 1'); SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]); diff --git a/client/test/sql/30_namedplaces_test.sql b/client/test/sql/30_namedplaces_test.sql index f973b94..73bbe9a 100644 --- a/client/test/sql/30_namedplaces_test.sql +++ b/client/test/sql/30_namedplaces_test.sql @@ -35,13 +35,13 @@ SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_namedplace_point('Elx'); SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_namedplace_point('Elx'); SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); diff --git a/client/test/sql/40_postalcodes_test.sql b/client/test/sql/40_postalcodes_test.sql index 707ebfa..8b2472c 100644 --- a/client/test/sql/40_postalcodes_test.sql +++ b/client/test/sql/40_postalcodes_test.sql @@ -26,12 +26,12 @@ SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); SELECT cdb_geocode_postalcode_point('03204', 'Spain'); diff --git a/client/test/sql/50_ipaddresses_test.sql b/client/test/sql/50_ipaddresses_test.sql index 36c8b96..c16de38 100644 --- a/client/test/sql/50_ipaddresses_test.sql +++ b/client/test/sql/50_ipaddresses_test.sql @@ -18,11 +18,11 @@ $$ LANGUAGE 'plpgsql'; SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -- Remove permissions diff --git a/client/test/sql/60_street_test.sql b/client/test/sql/60_street_test.sql index e47c124..a04d1d9 100644 --- a/client/test/sql/60_street_test.sql +++ b/client/test/sql/60_street_test.sql @@ -27,7 +27,7 @@ SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -- Grant other permissions but geocoding -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); SELECT cdb_geocode_street_point('One street, 1'); SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); @@ -40,7 +40,7 @@ SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); -- Grant geocoding permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}'); SELECT cdb_geocode_street_point('One street, 1'); SELECT cdb_geocode_street_point('One street', 'city'); SELECT cdb_geocode_street_point('One street', 'city', 'state'); diff --git a/client/test/sql/80_route_point_to_point_test.sql b/client/test/sql/80_route_point_to_point_test.sql index bff1c6d..018a148 100644 --- a/client/test/sql/80_route_point_to_point_test.sql +++ b/client/test/sql/80_route_point_to_point_test.sql @@ -39,7 +39,7 @@ SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POI SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -- Grant other permissions but routing -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "isolines"]}'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); @@ -48,7 +48,7 @@ SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POI SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles'); -- Grant routing permissions -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing"]}'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index 1508ba2..ea62696 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -227,6 +227,8 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); + -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); @@ -252,3 +254,4 @@ SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); SELECT obs_metadatavalidation(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001"}]', 1000); +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/95_data_observatory_tables_test.sql b/client/test/sql/95_data_observatory_tables_test.sql index 71054ad..6cf401c 100644 --- a/client/test/sql/95_data_observatory_tables_test.sql +++ b/client/test/sql/95_data_observatory_tables_test.sql @@ -34,6 +34,8 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); + -- Create a sample user table CREATE TABLE user_table (cartodb_id int, the_geom geometry); INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440'); @@ -53,4 +55,6 @@ SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'm SELECT * FROM my_table_dst; -- Clean tables -DROP TABLE my_table_dst; \ No newline at end of file +DROP TABLE my_table_dst; + +SELECT CDB_Conf_RemoveConf('api_keys_postgres'); diff --git a/client/test/sql/999_permissions_test.sql b/client/test/sql/999_permissions_test.sql index bac1254..2dbc6c0 100644 --- a/client/test/sql/999_permissions_test.sql +++ b/client/test/sql/999_permissions_test.sql @@ -35,7 +35,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 -- Grant other permissions but geocoding and routing SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["isolines"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["isolines"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); SELECT cdb_geocode_admin1_polygon('California'); @@ -51,7 +51,7 @@ SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87 -- Grant geocoding permissions SET ROLE postgres; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"permissions": ["geocoding", "routing"]}'); +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing"]}'); SET ROLE test_regular_user; -- Use regular user role SELECT cdb_geocode_admin0_polygon('Spain'); SELECT cdb_geocode_admin1_polygon('California'); From ad9619bb00b5ec4ec7b53122a48987a4ff3f18bf Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 12:04:19 +0200 Subject: [PATCH 214/265] Removed unused declaration --- client/sql/15_config_management.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/client/sql/15_config_management.sql b/client/sql/15_config_management.sql index 58cd1ea..304a2e9 100644 --- a/client/sql/15_config_management.sql +++ b/client/sql/15_config_management.sql @@ -16,7 +16,6 @@ DECLARE result cdb_dataservices_client._entity_config; apikey_config json; is_organization boolean; - username text; organization_name text DEFAULT NULL; BEGIN SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; From 53070781713a6a11f8f0c25d5dc06061c69566d2 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 12:13:06 +0200 Subject: [PATCH 215/265] Added CR suggestions --- .../templates/25_exception_safe_private_functions.erb | 4 ++-- client/sql/21_bulk_geocoding_functions.sql | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index 4a12b05..c027253 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -12,9 +12,9 @@ DECLARE _pg_exception_context TEXT; <% unless superuser_function? -%>apikey_permissions json;<% end %> BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + <% unless requires_permission %>IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; - END IF; + END IF;<% end %> <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index d4d7c5d..723ead3 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -17,9 +17,6 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; From 5989cb1dffb1a824acbe3b27309b874bee826a89 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 12:17:28 +0200 Subject: [PATCH 216/265] Fixed exception safe functions --- .../templates/25_exception_safe_private_functions.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index c027253..42f4f8b 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -18,8 +18,8 @@ BEGIN <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN RAISE EXCEPTION '<%= permission_error %>'; - END IF; - <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %><% end %> + END IF;<% end %> + <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %> -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From 3990d13572c84ed4589f22c8dac74e5d92bdbbd1 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 14:35:02 +0200 Subject: [PATCH 217/265] Server changes --- ...db_dataservices_server--0.33.0--0.34.0.sql | 425 ++ ...db_dataservices_server--0.34.0--0.33.0.sql | 412 ++ .../cdb_dataservices_server--0.34.0.sql | 3800 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.32.0--0.33.0.sql | 0 ...db_dataservices_server--0.33.0--0.32.0.sql | 0 .../cdb_dataservices_server--0.33.0.sql | 0 7 files changed, 4638 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.34.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.32.0--0.33.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.33.0--0.32.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.33.0.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql b/server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql new file mode 100644 index 0000000..8c3ede9 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql @@ -0,0 +1,425 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql b/server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql new file mode 100644 index 0000000..e74ed22 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql @@ -0,0 +1,412 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.33.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.0.sql new file mode 100644 index 0000000..80b1900 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.0.sql @@ -0,0 +1,3800 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index fb4506e..ecc5c9b 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.33.0' +default_version = '0.34.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.32.0--0.33.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.32.0--0.33.0.sql diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.33.0--0.32.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.33.0--0.32.0.sql diff --git a/server/extension/cdb_dataservices_server--0.33.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.33.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.33.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.33.0.sql From ffa67fe6ca65ec35de892f959a08929a71d93d7b Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 14:50:11 +0200 Subject: [PATCH 218/265] Client changes --- ...db_dataservices_client--0.25.0--0.26.0.sql | 4563 ++++++++++++++ ...db_dataservices_client--0.26.0--0.25.0.sql | 4175 ++++++++++++ client/cdb_dataservices_client--0.26.0.sql | 5606 +++++++++++++++++ client/cdb_dataservices_client.control | 2 +- ...db_dataservices_client--0.24.0--0.25.0.sql | 0 ...db_dataservices_client--0.25.0--0.24.0.sql | 0 .../cdb_dataservices_client--0.25.0.sql | 0 7 files changed, 14345 insertions(+), 1 deletion(-) create mode 100644 client/cdb_dataservices_client--0.25.0--0.26.0.sql create mode 100644 client/cdb_dataservices_client--0.26.0--0.25.0.sql create mode 100644 client/cdb_dataservices_client--0.26.0.sql rename client/{ => old_versions}/cdb_dataservices_client--0.24.0--0.25.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.25.0--0.24.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.25.0.sql (100%) diff --git a/client/cdb_dataservices_client--0.25.0--0.26.0.sql b/client/cdb_dataservices_client--0.25.0--0.26.0.sql new file mode 100644 index 0000000..9f4ba17 --- /dev/null +++ b/client/cdb_dataservices_client--0.25.0--0.26.0.sql @@ -0,0 +1,4563 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.0'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade + +DROP TYPE IF EXISTS cdb_dataservices_client._entity_config; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text, + apikey_permissions json +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + apikey_config json; + is_organization boolean; + organization_name text DEFAULT NULL; +BEGIN + SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; + + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + END IF; + result.username = apikey_config->>'username'; + result.organization_name = organization_name; + result.apikey_permissions = apikey_config->'permissions'; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; diff --git a/client/cdb_dataservices_client--0.26.0--0.25.0.sql b/client/cdb_dataservices_client--0.26.0--0.25.0.sql new file mode 100644 index 0000000..07695aa --- /dev/null +++ b/client/cdb_dataservices_client--0.26.0--0.25.0.sql @@ -0,0 +1,4175 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.25.0'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade + +DROP TYPE IF EXISTS cdb_dataservices_client._entity_config; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; +BEGIN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; diff --git a/client/cdb_dataservices_client--0.26.0.sql b/client/cdb_dataservices_client--0.26.0.sql new file mode 100644 index 0000000..d5bae96 --- /dev/null +++ b/client/cdb_dataservices_client--0.26.0.sql @@ -0,0 +1,5606 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE;-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text, + apikey_permissions json +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + apikey_config json; + is_organization boolean; + organization_name text DEFAULT NULL; +BEGIN + SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; + + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + END IF; + result.username = apikey_config->>'username'; + result.organization_name = organization_name; + result.apikey_permissions = apikey_config->'permissions'; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- For the OBS_Meta functions +CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC +); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params) + ) + ) + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + # Create a new table with the required columns + plpy.execute('CREATE TABLE "{schema}".{table_name} ( ' + 'cartodb_id int, the_geom geometry, {columns_with_types} ' + ');' + .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute( + "SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params))) + + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [ + colnames_arr[i] + + ' ' + + coltypes_arr[i] for i in range( + 0, + len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + aliased_colname_list = ','.join( + ['result.' + name for name in colnames_arr]) + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute( + "SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, " + "{schema}::text, {dbname}::text, {table_name}::text);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + user_db_role=plpy.quote_literal(user_db_role), + schema=plpy.quote_literal(user_schema), + dbname=plpy.quote_literal(dbname), + table_name=plpy.quote_literal(table_name))) + + if ds_fdw_metadata[0]["schemaname"]: + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + else: + raise Exception('Error connecting dataset via FDW') + + # Create a new table with the required columns + plpy.execute( + 'INSERT INTO "{schema}".{analysis_table_name} ' + 'SELECT ut.cartodb_id, ut.the_geom, {colname_list} ' + 'FROM "{schema}".{table_name} ut ' + 'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, ' + '{function_name}::text, {params}::json) ' + 'AS result ({columns_with_types}, cartodb_id int) ' + 'ON result.cartodb_id = ut.cartodb_id;' .format( + schema=user_schema, + analysis_table_name=output_table_name, + colname_list=aliased_colname_list, + table_name=table_name, + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params), + columns_with_types=columns_with_types)) + + # Wipe user FDW data from the server + wiped = plpy.execute( + "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, " + "{server_table_name}::text, {fdw_server}::text)" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + fdw_server=plpy.quote_literal(server_name))) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text +)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata( + username text, + orgname text, + function_name text, + params json +) RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData( + username text, + orgname text, + table_schema text, + table_name text, + function_name text, + params json +) RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( + username text, + orgname text, + table_schema text, + table_name text, + server_name text +) RETURNS boolean AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, orgname, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, orgname, ip_address); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, orgname, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_with_waypoints (username, orgname, waypoints, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundary (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundaryid (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundarybyid (username, orgname, geometry_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasure (username, orgname, geom, measure_id, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasurebyid (username, orgname, geom_ref, measure_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomvals, params, merge); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json) +RETURNS TABLE(id text, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomrefs, params); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeta (username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_metadatavalidation (username, orgname, geom_extent, geom_type, params, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getcategory (username, orgname, geom, category_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, orgname, geom, name, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensuscategory (username, orgname, geom, name, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getpopulation (username, orgname, geom, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_search (username, orgname, search_term, relevant_boundary); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailableboundaries (username, orgname, geom, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_dumpversion (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablenumerators (username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, section_tags text[] DEFAULT ARRAY[]::TEXT[], subsection_tags text[] DEFAULT ARRAY[]::TEXT[], other_tags text[] DEFAULT ARRAY[]::TEXT[], ids text[] DEFAULT ARRAY[]::TEXT[], name text DEFAULT NULL, denom_id text DEFAULT '', geom_id text DEFAULT '', timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._obs_getnumerators (username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabledenominators (username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, timespan text DEFAULT NULL, number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablegeometries (username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabletimespans (username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_legacybuildermetadata (username, orgname, aggregate_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info (username text, orgname text) +RETURNS SETOF service_quota_info AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) +RETURNS BOOLEAN AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_enough_quota (username, orgname, service, input_size); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck( + source_query text, + parameters json +) RETURNS boolean AS $$ +DECLARE + errors text[]; + validator_errors text[]; + validator_error text; + valid boolean; + geoms record; +BEGIN + errors := (ARRAY[])::TEXT[]; + FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type, + bool_and(st_isvalid(the_geom)) as valid, + avg(st_npoints(the_geom)) as avg_vertex, + ST_SetSRID(ST_Extent(the_geom), 4326) as extent, + count(*)::INT as numgeoms + FROM (%s) as _source + GROUP BY ST_GeometryType(the_geom)', source_query) + LOOP + IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN + errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type)); + END IF; + + IF geoms.valid IS FALSE THEN + errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them'); + END IF; + + -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here + -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813 + IF geoms.avg_vertex > 1000 THEN + errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them'); + END IF; + + -- OBS specific part + EXECUTE 'SELECT valid, errors + FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)' + INTO valid, validator_errors + USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms; + IF valid is FALSE THEN + FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors + LOOP + errors := array_append(errors, validator_error); + END LOOP; + END IF; + END LOOP; + + IF CARDINALITY(errors) > 0 THEN + RAISE EXCEPTION '%', errors; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_with_waypoints(waypoints geometry(Point, 4326)[], mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom Geometry, category_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom Geometry, name text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom Geometry, name text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text ) TO publicuser; + + + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 3037739..31a8d5d 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.25.0' +default_version = '0.26.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/cdb_dataservices_client--0.24.0--0.25.0.sql b/client/old_versions/cdb_dataservices_client--0.24.0--0.25.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.24.0--0.25.0.sql rename to client/old_versions/cdb_dataservices_client--0.24.0--0.25.0.sql diff --git a/client/cdb_dataservices_client--0.25.0--0.24.0.sql b/client/old_versions/cdb_dataservices_client--0.25.0--0.24.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.25.0--0.24.0.sql rename to client/old_versions/cdb_dataservices_client--0.25.0--0.24.0.sql diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/old_versions/cdb_dataservices_client--0.25.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.25.0.sql rename to client/old_versions/cdb_dataservices_client--0.25.0.sql From bafa60303c27032344eb56309dcc83528cc00243 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Sep 2018 15:45:06 +0200 Subject: [PATCH 219/265] Added data observatory permission --- ...db_dataservices_client--0.25.0--0.26.0.sql | 480 +++++++++--------- client/cdb_dataservices_client--0.26.0.sql | 480 +++++++++--------- client/renderer/interface.yaml | 90 ++++ .../expected/90_data_observatory_test.out | 105 +++- client/test/sql/90_data_observatory_test.sql | 56 +- 5 files changed, 727 insertions(+), 484 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0--0.26.0.sql b/client/cdb_dataservices_client--0.25.0--0.26.0.sql index 9f4ba17..e7ad49b 100644 --- a/client/cdb_dataservices_client--0.25.0--0.26.0.sql +++ b/client/cdb_dataservices_client--0.25.0--0.26.0.sql @@ -1021,11 +1021,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1049,11 +1049,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1077,11 +1077,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1105,11 +1105,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1133,11 +1133,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1161,11 +1161,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1189,11 +1189,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1217,11 +1217,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1245,11 +1245,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1273,11 +1273,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1301,11 +1301,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1329,11 +1329,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1357,11 +1357,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1385,11 +1385,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1413,11 +1413,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1441,11 +1441,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1469,11 +1469,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1497,11 +1497,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1525,11 +1525,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1553,11 +1553,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1581,11 +1581,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1609,11 +1609,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1637,11 +1637,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1665,11 +1665,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1693,11 +1693,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1721,11 +1721,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1749,11 +1749,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1777,11 +1777,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1805,11 +1805,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1833,11 +1833,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3142,11 +3142,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3181,11 +3181,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3220,11 +3220,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3259,11 +3259,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3298,11 +3298,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3337,11 +3337,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3376,11 +3376,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3415,11 +3415,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3454,11 +3454,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3493,11 +3493,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3532,11 +3532,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3571,11 +3571,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3610,11 +3610,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3649,11 +3649,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3688,11 +3688,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3727,11 +3727,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3766,11 +3766,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3805,11 +3805,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3844,11 +3844,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3883,11 +3883,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3922,11 +3922,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3961,11 +3961,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4000,11 +4000,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4039,11 +4039,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4078,11 +4078,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4117,11 +4117,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4156,11 +4156,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4195,11 +4195,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4234,11 +4234,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4273,11 +4273,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN diff --git a/client/cdb_dataservices_client--0.26.0.sql b/client/cdb_dataservices_client--0.26.0.sql index d5bae96..f3f8201 100644 --- a/client/cdb_dataservices_client--0.26.0.sql +++ b/client/cdb_dataservices_client--0.26.0.sql @@ -923,11 +923,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -951,11 +951,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -979,11 +979,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1007,11 +1007,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1035,11 +1035,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1063,11 +1063,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1091,11 +1091,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1119,11 +1119,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1147,11 +1147,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1175,11 +1175,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1203,11 +1203,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1231,11 +1231,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1259,11 +1259,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1287,11 +1287,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1315,11 +1315,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1343,11 +1343,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1371,11 +1371,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1399,11 +1399,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1427,11 +1427,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1455,11 +1455,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1483,11 +1483,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1511,11 +1511,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1539,11 +1539,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1567,11 +1567,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1595,11 +1595,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1623,11 +1623,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1651,11 +1651,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1679,11 +1679,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1707,11 +1707,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -1735,11 +1735,11 @@ DECLARE orgname text; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3407,11 +3407,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3446,11 +3446,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3485,11 +3485,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3524,11 +3524,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3563,11 +3563,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3602,11 +3602,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3641,11 +3641,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3680,11 +3680,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3719,11 +3719,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3758,11 +3758,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3797,11 +3797,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3836,11 +3836,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3875,11 +3875,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3914,11 +3914,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3953,11 +3953,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -3992,11 +3992,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4031,11 +4031,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4070,11 +4070,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4109,11 +4109,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4148,11 +4148,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4187,11 +4187,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4226,11 +4226,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4265,11 +4265,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4304,11 +4304,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4343,11 +4343,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4382,11 +4382,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4421,11 +4421,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4460,11 +4460,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4499,11 +4499,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN @@ -4538,11 +4538,11 @@ DECLARE _pg_exception_context TEXT; apikey_permissions json; BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 06d57e6..7bb0b69 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -302,6 +302,9 @@ - name: obs_get_demographic_snapshot return_type: json + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: time_span, type: "text", default: "'2009 - 2013'::text" } @@ -309,6 +312,9 @@ - name: obs_get_segment_snapshot return_type: json + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: geometry_level, type: text, default: 'NULL' } @@ -316,6 +322,9 @@ - name: obs_getdemographicsnapshot return_type: SETOF JSON multi_row: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: time_span, type: "text", default: 'NULL' } @@ -324,12 +333,18 @@ - name: obs_getsegmentsnapshot return_type: SETOF JSON multi_row: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: geometry_level, type: text, default: 'NULL' } - name: obs_getboundary return_type: Geometry + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: boundary_id, type: text } @@ -337,6 +352,9 @@ - name: obs_getboundaryid return_type: text + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: boundary_id, type: text } @@ -344,6 +362,9 @@ - name: obs_getboundarybyid return_type: Geometry + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geometry_id, type: text } - { name: boundary_id, type: text } @@ -353,6 +374,9 @@ return_type: TABLE(the_geom geometry, geom_refs text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: the_geom, type: geometry } - { name: geom_refs, type: text } @@ -366,6 +390,9 @@ return_type: TABLE(the_geom geometry, geom_refs text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: the_geom, type: geometry } - { name: geom_refs, type: text } @@ -380,6 +407,9 @@ return_type: TABLE(the_geom geometry, geom_refs text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: the_geom, type: geometry } - { name: geom_refs, type: text } @@ -393,6 +423,9 @@ return_type: TABLE(the_geom geometry, geom_refs text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: the_geom, type: geometry } - { name: geom_refs, type: text } @@ -405,6 +438,9 @@ - name: obs_getmeasure return_type: numeric + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: Geometry } - { name: measure_id, type: text } @@ -414,6 +450,9 @@ - name: obs_getmeasurebyid return_type: numeric + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom_ref, type: text } - { name: measure_id, type: text } @@ -424,6 +463,9 @@ return_type: TABLE(id int, data json) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: id, type: integer } - { name: data, type: json } @@ -436,6 +478,9 @@ return_type: TABLE(id text, data json) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: id, type: text } - { name: data, type: json } @@ -445,6 +490,9 @@ - name: obs_getmeta return_type: json + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom_ref, type: "Geometry(Geometry, 4326)" } - { name: params, type: json } @@ -456,6 +504,9 @@ return_type: TABLE(valid boolean, errors text[]) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom_extent, type: "Geometry(Geometry, 4326)" } - { name: geom_type, type: text } @@ -464,6 +515,9 @@ - name: obs_getcategory return_type: text + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: Geometry } - { name: category_id, type: text } @@ -472,6 +526,9 @@ - name: obs_getuscensusmeasure return_type: numeric + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: Geometry } - { name: name, type: text } @@ -481,6 +538,9 @@ - name: obs_getuscensuscategory return_type: text + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: Geometry } - { name: name, type: text } @@ -489,6 +549,9 @@ - name: obs_getpopulation return_type: numeric + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: geom, type: Geometry } - { name: normalize, type: text, default: 'NULL'} @@ -499,6 +562,9 @@ return_type: TABLE(id text, description text, name text, aggregate text, source text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: id, type: text } - { name: description, type: text } @@ -513,6 +579,9 @@ return_type: TABLE(boundary_id text, description text, time_span text, tablename text) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied table_fields: - { name: boundary_id, type: text } - { name: description, type: text } @@ -524,6 +593,9 @@ - name: obs_dumpversion return_type: text + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - {} @@ -531,6 +603,9 @@ return_type: SETOF cdb_dataservices_client.obs_meta_numerator multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' } - { name: filter_tags, type: "text[]", default: 'NULL' } @@ -542,6 +617,9 @@ return_type: SETOF cdb_dataservices_client.obs_meta_numerator multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' } - { name: section_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' } @@ -557,6 +635,9 @@ return_type: SETOF cdb_dataservices_client.obs_meta_denominator multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' } - { name: filter_tags, type: "text[]", default: 'NULL' } @@ -568,6 +649,9 @@ return_type: SETOF cdb_dataservices_client.obs_meta_geometry multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' } - { name: filter_tags, type: "text[]", default: 'NULL' } @@ -580,6 +664,9 @@ return_type: SETOF cdb_dataservices_client.obs_meta_timespan multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' } - { name: filter_tags, type: "text[]", default: 'NULL' } @@ -591,6 +678,9 @@ return_type: TABLE(name text, subsection json) multi_row: true multi_field: true + requires_permission: true + permission_name: observatory + permission_error: Data Observatory permission denied params: - { name: aggregate_type, type: text, default: 'NULL' } diff --git a/client/test/expected/90_data_observatory_test.out b/client/test/expected/90_data_observatory_test.out index 05b429d..f10e1a0 100644 --- a/client/test/expected/90_data_observatory_test.out +++ b/client/test/expected/90_data_observatory_test.out @@ -202,13 +202,114 @@ BEGIN RETURN QUERY SELECT true AS valid, ARRAY[]::TEXT[] AS errors; END; $$ LANGUAGE 'plpgsql'; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); +-- -- Exercise the public and the proxied function +-- No permissions granted +SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getsegmentsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundary(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundaryid(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundarybyid('36047'::text, 'us.census.tiger.county'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundariesbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500, 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text); +ERROR: Data Observatory permission denied +SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text); +ERROR: Data Observatory permission denied +SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text); +ERROR: Data Observatory permission denied +SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text); +ERROR: Data Observatory permission denied +SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'Spielman-Singleton Segments: 10 Clusters'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +ERROR: Data Observatory permission denied +SELECT obs_search('total_pop'::text); +ERROR: Data Observatory permission denied +SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +ERROR: Data Observatory permission denied +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +ERROR: Data Observatory permission denied +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +ERROR: Data Observatory permission denied +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +ERROR: Data Observatory permission denied +SELECT obs_metadatavalidation(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001"}]', 1000); +ERROR: Data Observatory permission denied +-- Grant other permissions but DO +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getsegmentsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundary(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundaryid(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundarybyid('36047'::text, 'us.census.tiger.county'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundariesbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500, 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text); +ERROR: Data Observatory permission denied +SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text); +ERROR: Data Observatory permission denied +SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text); +ERROR: Data Observatory permission denied +SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text); +ERROR: Data Observatory permission denied +SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text); +ERROR: Data Observatory permission denied +SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'Spielman-Singleton Segments: 10 Clusters'::text); +ERROR: Data Observatory permission denied +SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +ERROR: Data Observatory permission denied +SELECT obs_search('total_pop'::text); +ERROR: Data Observatory permission denied +SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +ERROR: Data Observatory permission denied +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +ERROR: Data Observatory permission denied +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +ERROR: Data Observatory permission denied +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +ERROR: Data Observatory permission denied +SELECT obs_metadatavalidation(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001"}]', 1000); +ERROR: Data Observatory permission denied +-- Grant DO permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["observatory"]}'); cdb_conf_setconf ------------------ (1 row) --- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group) obs_get_demographic_snapshot diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index ea62696..c46c2bd 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -227,9 +227,61 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": [""]}'); +-- -- Exercise the public and the proxied function --- Exercise the public and the proxied function +-- No permissions granted +SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); +SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); +SELECT obs_getsegmentsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +SELECT obs_getboundary(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundaryid(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundarybyid('36047'::text, 'us.census.tiger.county'::text); +SELECT obs_getboundariesbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500, 'us.census.tiger.census_tract'::text); +SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text); +SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text); +SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text); +SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text); +SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text); +SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'Spielman-Singleton Segments: 10 Clusters'::text); +SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_search('total_pop'::text); +SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +SELECT obs_metadatavalidation(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001"}]', 1000); + +-- Grant other permissions but DO +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}'); +SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); +SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); +SELECT obs_getsegmentsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); +SELECT obs_getboundary(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundaryid(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundarybyid('36047'::text, 'us.census.tiger.county'::text); +SELECT obs_getboundariesbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500, 'us.census.tiger.census_tract'::text); +SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text); +SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text); +SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text); +SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text); +SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text); +SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text); +SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'Spielman-Singleton Segments: 10 Clusters'::text); +SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_search('total_pop'::text); +SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +SELECT obs_metadatavalidation(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001"}]', 1000); + +-- Grant DO permissions +SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["observatory"]}'); SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); From 88e6245014b9eac73ab73a09a8d830be4a292d4f Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 13 Sep 2018 10:00:32 +0200 Subject: [PATCH 220/265] Updated NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 9de5aaa..ca4e3d5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +Sep 13th, 2018 +============== +* Version `0.34.0` of the server, and `0.26.0` of the client. + * Use dataservices with non-master keys Aug 27th, 2018 ============== From a57500c966af897abf66846af4193a35c546d5fd Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Thu, 13 Sep 2018 16:01:53 +0200 Subject: [PATCH 221/265] Fixed tests --- test/integration/test_admin0_functions.py | 5 +- test/integration/test_admin1_functions.py | 15 +++-- .../test_data_observatory_functions.py | 55 +++++++++---------- test/integration/test_ipaddress_functions.py | 11 ++-- test/integration/test_isolines_functions.py | 7 +-- test/integration/test_namedplace_functions.py | 19 +++---- test/integration/test_postalcode_functions.py | 15 +++-- test/integration/test_routing_functions.py | 12 ++-- test/integration/test_street_functions.py | 3 +- 9 files changed, 67 insertions(+), 75 deletions(-) diff --git a/test/integration/test_admin0_functions.py b/test/integration/test_admin0_functions.py index e6dbe27..9d3dd79 100644 --- a/test/integration/test_admin0_functions.py +++ b/test/integration/test_admin0_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,7 +12,6 @@ class TestAdmin0Functions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_admin0_is_ok(self): @@ -29,4 +28,4 @@ class TestAdmin0Functions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_admin0_polygon(text) does not exist"]) diff --git a/test/integration/test_admin1_functions.py b/test/integration/test_admin1_functions.py index 0aca567..edfe19e 100644 --- a/test/integration/test_admin1_functions.py +++ b/test/integration/test_admin1_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,30 +12,29 @@ class TestAdmin1Functions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_admin1_without_country_is_ok(self): query = "SELECT cdb_geocode_admin1_polygon(province) as geometry " \ "FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_admin1_with_country_is_ok(self): query = "SELECT cdb_geocode_admin1_polygon(province,country)" \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_admin1_without_api_key_raise_error(self): query = "SELECT cdb_geocode_admin1_polygon(province) as geometry " \ "FROM {0} LIMIT 1".format( - self.env_variables['table_name']) + self.env_variables['table_name']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_admin1_polygon(text) does not exist"]) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 105483f..00e916b 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,7 +12,6 @@ class TestDataObservatoryFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_get_demographic_snapshot_is_ok(self): @@ -25,7 +24,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"]) def test_if_get_segment_snapshot_is_ok(self): query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) @@ -37,7 +36,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"]) def test_if_get_measure_with_point_is_ok(self): query = "SELECT OBS_GetMeasure(CDB_LatLng(40.704512, -73.936669), 'us.census.acs.B01003001') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -54,7 +53,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"]) def test_if_get_measure_by_id_ok(self): query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -66,7 +65,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"]) def test_if_get_category_is_ok(self): query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -78,7 +77,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"]) def test_if_get_us_census_measure_with_point_is_ok(self): query = "SELECT OBS_GetUSCensusMeasure(CDB_LatLng(40.704512, -73.936669), 'male population') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -95,7 +94,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"]) def test_if_get_us_census_category_is_ok(self): query = "SELECT OBS_GetUSCensusCategory('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'Spielman-Singleton Segments: 10 Clusters') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -107,7 +106,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"]) def test_if_get_population_is_ok(self): query = "SELECT OBS_GetPopulation(CDB_LatLng(40.704512, -73.936669)) as population;&api_key={0}".format(self.env_variables['api_key']) @@ -119,7 +118,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpopulation(geometry) does not exist"]) def test_if_obs_search_is_ok(self): sql = "SELECT id FROM OBS_Search('total_pop') WHERE id LIKE 'es.ine%' LIMIT 1;" @@ -133,7 +132,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_search(unknown) does not exist"]) def test_if_obs_get_available_boundaries_is_ok(self): query = "SELECT boundary_id FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.704512, -73.936669)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -145,7 +144,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"]) def test_if_obs_get_boundary_is_ok(self): query = "SELECT OBS_GetBoundary(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -157,7 +156,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"]) def test_if_obs_get_boundary_id_is_ok(self): query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) @@ -169,7 +168,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"]) def test_if_obs_get_boundary_by_id_is_ok(self): query = "SELECT OBS_GetBoundaryById('36047', 'us.census.tiger.county', '2014') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -181,7 +180,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"]) def test_if_obs_get_boundaries_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -193,7 +192,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_boundaries_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -205,7 +204,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"]) def test_if_obs_get_points_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -217,7 +216,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_points_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract', '2014') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -229,19 +228,19 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"]) def test_if_obs_get_legacy_builder_metadata_is_ok(self): query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) result = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(result['name'], None) - def test_if_obs_get_points_by_point_and_radius_without_api_key_raise_error(self): + def test_if_legacy_builder_metadata_without_api_key_raise_error(self): query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;" try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_legacybuildermetadata() does not exist"]) def test_if_obs_get_available_numerators_is_ok(self): query = "SELECT numer_id FROM OBS_GetAvailableNumerators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -253,7 +252,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailablenumerators() does not exist"]) def test_if_obs_get_available_denominators_is_ok(self): query = "SELECT denom_id FROM OBS_GetAvailableDenominators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -265,7 +264,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailabledenominators() does not exist"]) def test_if_obs_get_available_geometries_is_ok(self): query = "SELECT geom_id FROM OBS_GetAvailableGeometries() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -277,7 +276,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailablegeometries() does not exist"]) def test_if_obs_get_available_timespans_is_ok(self): query = "SELECT timespan_id FROM OBS_GetAvailableTimespans() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -289,7 +288,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailabletimespans() does not exist"]) def test_if_obs_get_meta_is_ok(self): params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\'' @@ -303,7 +302,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) def test_if_obs_get_data_is_ok(self): params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\'' @@ -323,8 +322,8 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query_1) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) try: IntegrationTestHelper.execute_query(self.sql_api_url, query_2) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"]) diff --git a/test/integration/test_ipaddress_functions.py b/test/integration/test_ipaddress_functions.py index e257541..5a06916 100644 --- a/test/integration/test_ipaddress_functions.py +++ b/test/integration/test_ipaddress_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,22 +12,21 @@ class TestPostalcodeFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_ipaddress_point_is_ok(self): query = "SELECT cdb_geocode_ipaddress_point(ip) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_ipaddress_without_api_key_raise_error(self): query = "SELECT cdb_geocode_ipaddress_point(ip) " \ "as geometry FROM {0} LIMIT 1".format( - self.env_variables['table_name']) + self.env_variables['table_name']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_ipaddress_point(text) does not exist"]) diff --git a/test/integration/test_isolines_functions.py b/test/integration/test_isolines_functions.py index 3e322d0..6622b40 100644 --- a/test/integration/test_isolines_functions.py +++ b/test/integration/test_isolines_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,7 +12,6 @@ class TestIsolinesFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_isochrones_is_ok(self): @@ -28,7 +27,7 @@ class TestIsolinesFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Isolines permission denied", "function cdb_isochrone(geometry, unknown, integer[]) does not exist"]) def test_if_select_with_isodistance_is_ok(self): query = "SELECT * FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, " \ @@ -43,4 +42,4 @@ class TestIsolinesFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Isolines permission denied", "function cdb_isodistance(geometry, unknown, integer[]) does not exist"]) diff --git a/test/integration/test_namedplace_functions.py b/test/integration/test_namedplace_functions.py index 008b6e4..6ed87d4 100644 --- a/test/integration/test_namedplace_functions.py +++ b/test/integration/test_namedplace_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,38 +12,37 @@ class TestNameplaceFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_namedplace_city_is_ok(self): query = "SELECT cdb_geocode_namedplace_point(city) as geometry " \ "FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_namedplace_city_country_is_ok(self): query = "SELECT cdb_geocode_namedplace_point(city,country) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_namedplace_city_province_country_is_ok(self): query = "SELECT cdb_geocode_namedplace_point(city,province,country) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_namedplace_without_api_key_raise_error(self): query = "SELECT cdb_geocode_namedplace_point(city) as geometry " \ "FROM {0} LIMIT 1".format( - self.env_variables['table_name']) + self.env_variables['table_name']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_namedplace_point(text) does not exist"]) diff --git a/test/integration/test_postalcode_functions.py b/test/integration/test_postalcode_functions.py index 942758d..909d4a3 100644 --- a/test/integration/test_postalcode_functions.py +++ b/test/integration/test_postalcode_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,30 +12,29 @@ class TestPostalcodeFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_postalcode_polygon_is_ok(self): query = "SELECT cdb_geocode_postalcode_polygon(postalcode, country) " \ "as geometry FROM {0} WHERE country='USA'&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_postalcode_point_is_ok(self): query = "SELECT cdb_geocode_postalcode_point(postalcode, country) " \ "as geometry FROM {0} WHERE country='Spain'&api_key={1}".format( - self.env_variables['table_name'], - self.env_variables['api_key']) + self.env_variables['table_name'], + self.env_variables['api_key']) geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) def test_if_select_with_postalcode_without_api_key_raise_error(self): query = "SELECT cdb_geocode_postalcode_polygon(postalcode, country) " \ "as geometry FROM {0} WHERE country = 'USA'".format( - self.env_variables['table_name']) + self.env_variables['table_name']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_postalcode_polygon(text, text) does not exist"]) \ No newline at end of file diff --git a/test/integration/test_routing_functions.py b/test/integration/test_routing_functions.py index 6d38e00..d6a9ea9 100644 --- a/test/integration/test_routing_functions.py +++ b/test/integration/test_routing_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper @@ -12,7 +12,6 @@ class TestRoutingFunctions(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) def test_if_select_with_routing_point_to_point_is_ok(self): @@ -32,7 +31,7 @@ class TestRoutingFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Routing permission denied", "function cdb_route_point_to_point(geometry, geometry, unknown, text[]) does not exist"]) def test_if_select_with_routing_with_waypoints_is_ok(self): query = "SELECT duration, length, shape as the_geom " \ @@ -46,11 +45,12 @@ class TestRoutingFunctions(TestCase): def test_if_select_with_routing_with_waypoints_without_api_key_raise_error(self): query = "SELECT duration, length, shape as the_geom " \ - "FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::GEOMETRY, "\ + "FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::geometry, "\ "'POINT(-3.7059 40.4203)'::geometry, 'POINT(-3.7046 40.4180)'::geometry]" \ "::geometry[], 'car', " \ - "ARRAY['mode_type=shortest']::text[])&api_key={0}" + "ARRAY['mode_type=shortest']::text[])&api_key={0}".format( + self.env_variables['api_key']) try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") + assert_true(e.message[0] in ["Routing permission denied", "function cdb_route_with_waypoints(geometry, geometry, text, text[]) does not exist"]) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index b7557c9..fd1b983 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -110,7 +110,6 @@ class TestStreetFunctionsSetUp(TestCase): self.env_variables['schema'], self.env_variables['username'], self.env_variables['host'], - self.env_variables['api_key'] ) if not self.fixture_points: @@ -123,7 +122,6 @@ class TestStreetFunctionsSetUp(TestCase): self.metadata = self.METADATAS[provider] - def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query, self.env_variables[ @@ -137,6 +135,7 @@ class TestStreetFunctionsSetUp(TestCase): "where service = 'hires_geocoder'" return self._run_authenticated(query)['rows'][0]['used_quota'] + class TestStreetFunctions(TestStreetFunctionsSetUp): def test_if_select_with_street_point_is_ok(self): From 4598d5ed7df56d164c59aa0a9e332e7f5be8378f Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Fri, 14 Sep 2018 10:14:59 +0200 Subject: [PATCH 222/265] Added CR suggestion --- test/integration/test_admin0_functions.py | 4 +- test/integration/test_admin1_functions.py | 4 +- .../test_data_observatory_functions.py | 52 +++++++++---------- test/integration/test_ipaddress_functions.py | 4 +- test/integration/test_isolines_functions.py | 6 +-- test/integration/test_namedplace_functions.py | 4 +- test/integration/test_postalcode_functions.py | 4 +- test/integration/test_routing_functions.py | 6 +-- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/test/integration/test_admin0_functions.py b/test/integration/test_admin0_functions.py index 9d3dd79..7e98abe 100644 --- a/test/integration/test_admin0_functions.py +++ b/test/integration/test_admin0_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -28,4 +28,4 @@ class TestAdmin0Functions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_admin0_polygon(text) does not exist"]) + assert_in(e.message[0], ["Geocoding permission denied", "function cdb_geocode_admin0_polygon(text) does not exist"]) diff --git a/test/integration/test_admin1_functions.py b/test/integration/test_admin1_functions.py index edfe19e..92ffb41 100644 --- a/test/integration/test_admin1_functions.py +++ b/test/integration/test_admin1_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -37,4 +37,4 @@ class TestAdmin1Functions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_admin1_polygon(text) does not exist"]) + assert_in(e.message[0], ["Geocoding permission denied", "function cdb_geocode_admin1_polygon(text) does not exist"]) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 00e916b..661678a 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -24,7 +24,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"]) def test_if_get_segment_snapshot_is_ok(self): query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) @@ -36,7 +36,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"]) def test_if_get_measure_with_point_is_ok(self): query = "SELECT OBS_GetMeasure(CDB_LatLng(40.704512, -73.936669), 'us.census.acs.B01003001') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -53,7 +53,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"]) def test_if_get_measure_by_id_ok(self): query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -65,7 +65,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"]) def test_if_get_category_is_ok(self): query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -77,7 +77,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"]) def test_if_get_us_census_measure_with_point_is_ok(self): query = "SELECT OBS_GetUSCensusMeasure(CDB_LatLng(40.704512, -73.936669), 'male population') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -94,7 +94,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"]) def test_if_get_us_census_category_is_ok(self): query = "SELECT OBS_GetUSCensusCategory('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'Spielman-Singleton Segments: 10 Clusters') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -106,7 +106,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"]) def test_if_get_population_is_ok(self): query = "SELECT OBS_GetPopulation(CDB_LatLng(40.704512, -73.936669)) as population;&api_key={0}".format(self.env_variables['api_key']) @@ -118,7 +118,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpopulation(geometry) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getpopulation(geometry) does not exist"]) def test_if_obs_search_is_ok(self): sql = "SELECT id FROM OBS_Search('total_pop') WHERE id LIKE 'es.ine%' LIMIT 1;" @@ -132,7 +132,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_search(unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_search(unknown) does not exist"]) def test_if_obs_get_available_boundaries_is_ok(self): query = "SELECT boundary_id FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.704512, -73.936669)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -144,7 +144,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"]) def test_if_obs_get_boundary_is_ok(self): query = "SELECT OBS_GetBoundary(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -156,7 +156,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"]) def test_if_obs_get_boundary_id_is_ok(self): query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) @@ -168,7 +168,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"]) def test_if_obs_get_boundary_by_id_is_ok(self): query = "SELECT OBS_GetBoundaryById('36047', 'us.census.tiger.county', '2014') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -180,7 +180,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"]) def test_if_obs_get_boundaries_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -192,7 +192,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_boundaries_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -204,7 +204,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"]) def test_if_obs_get_points_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -216,7 +216,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_points_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract', '2014') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -228,7 +228,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"]) def test_if_obs_get_legacy_builder_metadata_is_ok(self): query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -240,7 +240,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_legacybuildermetadata() does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_legacybuildermetadata() does not exist"]) def test_if_obs_get_available_numerators_is_ok(self): query = "SELECT numer_id FROM OBS_GetAvailableNumerators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -252,7 +252,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailablenumerators() does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailablenumerators() does not exist"]) def test_if_obs_get_available_denominators_is_ok(self): query = "SELECT denom_id FROM OBS_GetAvailableDenominators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -264,7 +264,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailabledenominators() does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailabledenominators() does not exist"]) def test_if_obs_get_available_geometries_is_ok(self): query = "SELECT geom_id FROM OBS_GetAvailableGeometries() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -276,7 +276,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailablegeometries() does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailablegeometries() does not exist"]) def test_if_obs_get_available_timespans_is_ok(self): query = "SELECT timespan_id FROM OBS_GetAvailableTimespans() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -288,7 +288,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getavailabletimespans() does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailabletimespans() does not exist"]) def test_if_obs_get_meta_is_ok(self): params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\'' @@ -302,7 +302,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) def test_if_obs_get_data_is_ok(self): params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\'' @@ -322,8 +322,8 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query_1) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) try: IntegrationTestHelper.execute_query(self.sql_api_url, query_2) except Exception as e: - assert_true(e.message[0] in ["Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"]) diff --git a/test/integration/test_ipaddress_functions.py b/test/integration/test_ipaddress_functions.py index 5a06916..191d7d7 100644 --- a/test/integration/test_ipaddress_functions.py +++ b/test/integration/test_ipaddress_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -29,4 +29,4 @@ class TestPostalcodeFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_ipaddress_point(text) does not exist"]) + assert_in(e.message[0], ["Geocoding permission denied", "function cdb_geocode_ipaddress_point(text) does not exist"]) diff --git a/test/integration/test_isolines_functions.py b/test/integration/test_isolines_functions.py index 6622b40..8da7e00 100644 --- a/test/integration/test_isolines_functions.py +++ b/test/integration/test_isolines_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -27,7 +27,7 @@ class TestIsolinesFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Isolines permission denied", "function cdb_isochrone(geometry, unknown, integer[]) does not exist"]) + assert_in(e.message[0], ["Isolines permission denied", "function cdb_isochrone(geometry, unknown, integer[]) does not exist"]) def test_if_select_with_isodistance_is_ok(self): query = "SELECT * FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, " \ @@ -42,4 +42,4 @@ class TestIsolinesFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Isolines permission denied", "function cdb_isodistance(geometry, unknown, integer[]) does not exist"]) + assert_in(e.message[0], ["Isolines permission denied", "function cdb_isodistance(geometry, unknown, integer[]) does not exist"]) diff --git a/test/integration/test_namedplace_functions.py b/test/integration/test_namedplace_functions.py index 6ed87d4..6d64fd5 100644 --- a/test/integration/test_namedplace_functions.py +++ b/test/integration/test_namedplace_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -45,4 +45,4 @@ class TestNameplaceFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_namedplace_point(text) does not exist"]) + assert_in(e.message[0], ["Geocoding permission denied", "function cdb_geocode_namedplace_point(text) does not exist"]) diff --git a/test/integration/test_postalcode_functions.py b/test/integration/test_postalcode_functions.py index 909d4a3..13151af 100644 --- a/test/integration/test_postalcode_functions.py +++ b/test/integration/test_postalcode_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -37,4 +37,4 @@ class TestPostalcodeFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Geocoding permission denied", "function cdb_geocode_postalcode_polygon(text, text) does not exist"]) \ No newline at end of file + assert_in(e.message[0], ["Geocoding permission denied", "function cdb_geocode_postalcode_polygon(text, text) does not exist"]) \ No newline at end of file diff --git a/test/integration/test_routing_functions.py b/test/integration/test_routing_functions.py index d6a9ea9..6e9ad65 100644 --- a/test/integration/test_routing_functions.py +++ b/test/integration/test_routing_functions.py @@ -1,6 +1,6 @@ from unittest import TestCase from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_true +from nose.tools import assert_not_equal, assert_in from ..helpers.integration_test_helper import IntegrationTestHelper @@ -31,7 +31,7 @@ class TestRoutingFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Routing permission denied", "function cdb_route_point_to_point(geometry, geometry, unknown, text[]) does not exist"]) + assert_in(e.message[0], ["Routing permission denied", "function cdb_route_point_to_point(geometry, geometry, unknown, text[]) does not exist"]) def test_if_select_with_routing_with_waypoints_is_ok(self): query = "SELECT duration, length, shape as the_geom " \ @@ -53,4 +53,4 @@ class TestRoutingFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_true(e.message[0] in ["Routing permission denied", "function cdb_route_with_waypoints(geometry, geometry, text, text[]) does not exist"]) + assert_in(e.message[0], ["Routing permission denied", "function cdb_route_with_waypoints(geometry, geometry, text, text[]) does not exist"]) From 4aa587d32bbfb1e859cb2e138424d5c84fbad279 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Fri, 14 Sep 2018 12:46:43 +0200 Subject: [PATCH 223/265] Fixing DO integration tests --- .../test_data_observatory_functions.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 661678a..23305bf 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -24,7 +24,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getdemographicsnapshot(geometry) does not exist"]) def test_if_get_segment_snapshot_is_ok(self): query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) @@ -36,7 +36,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getsegmentsnapshot(geometry) does not exist"]) def test_if_get_measure_with_point_is_ok(self): query = "SELECT OBS_GetMeasure(CDB_LatLng(40.704512, -73.936669), 'us.census.acs.B01003001') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -53,7 +53,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeasure(geometry, unknown) does not exist"]) def test_if_get_measure_by_id_ok(self): query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -65,7 +65,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeasurebyid(unknown, unknown, unknown, unknown) does not exist"]) def test_if_get_category_is_ok(self): query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -77,7 +77,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getcategory(geometry, unknown, unknown, unknown) does not exist"]) def test_if_get_us_census_measure_with_point_is_ok(self): query = "SELECT OBS_GetUSCensusMeasure(CDB_LatLng(40.704512, -73.936669), 'male population') as measure;&api_key={0}".format(self.env_variables['api_key']) @@ -94,7 +94,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getuscensusmeasure(geometry, unknown) does not exist"]) def test_if_get_us_census_category_is_ok(self): query = "SELECT OBS_GetUSCensusCategory('0103000020E61000000100000021000000BB366F6D917B52C0E7BB6EA82B5A444067224C85937B52C0810205E70E5A4440596D6342997B52C09BED2952F35944400A96386CA27B52C07B6D31F9D95944402B81A0A8AE7B52C0D16B73D5C3594440C34C397FBD7B52C0D8B5B7C0B1594440CFD90A5ECE7B52C0BE8DD86CA45944405C7E229FE07B52C0A904EE5C9C59444017C1F28EF37B52C05E2745E099594440AE8A3873067C52C0FD62540F9D5944402F272292187C52C05502CBCAA5594440C3F17139297C52C0B3FBC4BCB3594440DDAE56C5377C52C00C46175CC6594440E96AB6A6437C52C0355C94F1DC5944402313AE684C7C52C05340159FF6594440B3B90FB5517C52C031F30168125A44407F43B357537C52C0DF95053B2F5A444059C17840517C52C0D9E08EFC4B5A44406580E8834B7C52C046B5B591675A444035736A5A427C52C0AF9A1AEB805A44402E721C1E367C52C0D226550F975A4440506D5C47277C52C0A2968A24A95A4440507C2868167C52C0D51FCE78B65A4440C4438226047C52C02369F888BE5A4440C9F10C36F17B52C027B1B205C15A4440AABE2451DE7B52C0F5E383D6BD5A44402A18B431CC7B52C01785C11AB55A444018320D8ABB7B52C070265B28A75A4440E28A0EFEAC7B52C09E518C88945A44401D08D61CA17B52C09E8195F27D5A4440D7C3405B987B52C0643CB044645A444047B16D0F937B52C04EC9837B485A4440BB366F6D917B52C0E7BB6EA82B5A4440'::geometry, 'Spielman-Singleton Segments: 10 Clusters') as category;&api_key={0}".format(self.env_variables['api_key']) @@ -106,7 +106,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getuscensuscategory(geometry, unknown) does not exist"]) def test_if_get_population_is_ok(self): query = "SELECT OBS_GetPopulation(CDB_LatLng(40.704512, -73.936669)) as population;&api_key={0}".format(self.env_variables['api_key']) @@ -118,7 +118,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getpopulation(geometry) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpopulation(geometry) does not exist"]) def test_if_obs_search_is_ok(self): sql = "SELECT id FROM OBS_Search('total_pop') WHERE id LIKE 'es.ine%' LIMIT 1;" @@ -132,7 +132,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_search(unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_search(unknown) does not exist"]) def test_if_obs_get_available_boundaries_is_ok(self): query = "SELECT boundary_id FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.704512, -73.936669)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -144,7 +144,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailableboundaries(geometry) does not exist"]) def test_if_obs_get_boundary_is_ok(self): query = "SELECT OBS_GetBoundary(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -156,7 +156,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundary(geometry, unknown) does not exist"]) def test_if_obs_get_boundary_id_is_ok(self): query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) @@ -168,7 +168,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundaryid(geometry, unknown, unknown) does not exist"]) def test_if_obs_get_boundary_by_id_is_ok(self): query = "SELECT OBS_GetBoundaryById('36047', 'us.census.tiger.county', '2014') as boundary;&api_key={0}".format(self.env_variables['api_key']) @@ -180,7 +180,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundarybyid(unknown, unknown, unknown) does not exist"]) def test_if_obs_get_boundaries_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -192,7 +192,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundariesbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_boundaries_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetBoundariesByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -204,7 +204,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getboundariesbypointandradius(geometry, integer, unknown) does not exist"]) def test_if_obs_get_points_by_geometry_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByGeometry(ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,4326), 'us.census.tiger.census_tract') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -216,7 +216,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpointsbygeometry(geometry, unknown) does not exist"]) def test_if_obs_get_points_by_point_and_radius_is_ok(self): query = "SELECT geom_refs FROM OBS_GetPointsByPointAndRadius(CDB_LatLng(40.704512, -73.936669), 500, 'us.census.tiger.census_tract', '2014') ORDER BY geom_refs ASC LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -228,7 +228,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getpointsbypointandradius(geometry, integer, unknown, unknown) does not exist"]) def test_if_obs_get_legacy_builder_metadata_is_ok(self): query = "SELECT name FROM OBS_LegacyBuilderMetadata() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -240,7 +240,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_legacybuildermetadata() does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_legacybuildermetadata() does not exist"]) def test_if_obs_get_available_numerators_is_ok(self): query = "SELECT numer_id FROM OBS_GetAvailableNumerators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -252,7 +252,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailablenumerators() does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailablenumerators() does not exist"]) def test_if_obs_get_available_denominators_is_ok(self): query = "SELECT denom_id FROM OBS_GetAvailableDenominators() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -264,7 +264,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailabledenominators() does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailabledenominators() does not exist"]) def test_if_obs_get_available_geometries_is_ok(self): query = "SELECT geom_id FROM OBS_GetAvailableGeometries() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -276,7 +276,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailablegeometries() does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailablegeometries() does not exist"]) def test_if_obs_get_available_timespans_is_ok(self): query = "SELECT timespan_id FROM OBS_GetAvailableTimespans() LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) @@ -288,7 +288,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getavailabletimespans() does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getavailabletimespans() does not exist"]) def test_if_obs_get_meta_is_ok(self): params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\'' @@ -302,7 +302,7 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) def test_if_obs_get_data_is_ok(self): params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\'' @@ -322,8 +322,8 @@ class TestDataObservatoryFunctions(TestCase): try: IntegrationTestHelper.execute_query(self.sql_api_url, query_1) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown, integer, integer, integer) does not exist"]) try: IntegrationTestHelper.execute_query(self.sql_api_url, query_2) except Exception as e: - assert_in(e.message[0], ["Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"]) + assert_in(e.message[0], ["Data Observatory permission denied", "function obs_getmeta(geometry, unknown) does not exist"]) From 61a27a9f9dea421c23c0fd486b93ffd3828fef55 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 3 Oct 2018 14:21:55 +0200 Subject: [PATCH 224/265] Tomtom QPS fix Tomtom return 403 status code in the following scenarios: - Forbidden - Not authorized - Account inactive - Account over queries per second limit - Account over rate limit - Rate limit exceeded So when we reach the QPS limit we receive a 403 instead of 429 so we have to include the provider in the qps service to differentiate between providers that returns 429 and special cases like tomtom --- .../cartodb_services/tomtom/isolines.py | 2 +- .../cartodb_services/tools/qps.py | 23 +++++++++---------- server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py index a6fc062..8a58f12 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py @@ -64,7 +64,7 @@ class TomTomIsolines(): return self._parse_coordinates(reachable_range[ENTRY_BOUNDARY]) - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def _calculate_isoline(self, origin, time_range, profile=DEFAULT_PROFILE, date_time=DEFAULT_DEPARTAT): diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index 074155b..ef210ab 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -18,15 +18,10 @@ def qps_retry(original_function=None, **options): """ if original_function is not None: def wrapped_function(*args, **kwargs): - if 'timeout' in options: - timeout = options['timeout'] - else: - timeout = DEFAULT_RETRY_TIMEOUT - if 'qps' in options: - qps = options['qps'] - else: - qps = DEFAULT_QUERIES_PER_SECOND - return QPSService(retry_timeout=timeout, queries_per_second=qps).call(original_function, *args, **kwargs) + timeout = options.get('timeout', DEFAULT_RETRY_TIMEOUT) + qps = options.get('qps', DEFAULT_QUERIES_PER_SECOND) + provider = options.get('provider', None) + return QPSService(retry_timeout=timeout, queries_per_second=qps, provider=provider).call(original_function, *args, **kwargs) return wrapped_function else: def partial_wrapper(func): @@ -36,9 +31,10 @@ def qps_retry(original_function=None, **options): class QPSService: - def __init__(self, queries_per_second, retry_timeout): + def __init__(self, queries_per_second, retry_timeout, provider): self._queries_per_second = queries_per_second self._retry_timeout = retry_timeout + self._provider = provider def call(self, fn, *args, **kwargs): start_time = datetime.now() @@ -48,8 +44,11 @@ class QPSService: return fn(*args, **kwargs) except Exception as e: response = getattr(e, 'response', None) - if response is not None and (response.status_code == 429): - self.retry(start_time, attempt_number) + if response is not None: + if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403): + self.retry(start_time, attempt_number) + elif response.status_code == 429: + self.retry(start_time, attempt_number) else: raise e attempt_number += 1 diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index ea9ab90..2693d0e 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.20.0', + version='0.20.1', description='CartoDB Services API Python Library', From 56a2ebb21045863ebf6105cdb7d732af571d5b27 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 3 Oct 2018 18:50:20 +0200 Subject: [PATCH 225/265] Fix qps timeout bug --- .../lib/python/cartodb_services/cartodb_services/tools/qps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index ef210ab..24d110e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -49,6 +49,8 @@ class QPSService: self.retry(start_time, attempt_number) elif response.status_code == 429: self.retry(start_time, attempt_number) + else: + raise e else: raise e attempt_number += 1 From cd8173c7e0bd35223e2bd9be311e1383e67fab7e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 23 Oct 2018 16:36:05 +0200 Subject: [PATCH 226/265] Include note to explain why some isodistances could not be precise --- doc/isoline_functions.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/isoline_functions.md b/doc/isoline_functions.md index 97949b9..743c589 100644 --- a/doc/isoline_functions.md +++ b/doc/isoline_functions.md @@ -16,6 +16,8 @@ The following functions provide an isoline generator service, based on time or d Displays a contoured line on a map, connecting geometries to a defined area, measured by an equal range of distance (in meters). +Note that not all the providers, for example TomTom, provide us a way to define the isoline limit in distance so we need to make some estimations. Due that estimations the produced isolines could not be 100% precise. + #### Arguments Name | Type | Description | Accepted values From 66409097803c16a200281c2e44dca42f0cc2db82 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:08 +0200 Subject: [PATCH 227/265] Add provider for QPS manger in tomtom services --- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 4 ++-- .../cartodb_services/cartodb_services/tomtom/routing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index b5ec592..f387bba 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -70,7 +70,7 @@ class TomTomGeocoder(Traceable): return False - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode(self, searchtext, city=None, state_province=None, country=None): response = self.geocode_meta(searchtext, city, state_province, country) @@ -80,7 +80,7 @@ class TomTomGeocoder(Traceable): else: return response[0] - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode_meta(self, searchtext, city=None, state_province=None, country=None): if searchtext: diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index dfc30fb..71dfa22 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -89,7 +89,7 @@ class TomTomRouting(Traceable): point[ENTRY_LONGITUDE])) return geometry - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def directions(self, waypoints, profile=DEFAULT_PROFILE, date_time=DEFAULT_DEPARTAT): self._validate_profile(profile) From 17c993f6ef16daa6497070aa14df781a5213324c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:44 +0200 Subject: [PATCH 228/265] Bump version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 2693d0e..02323b8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.20.1', + version='0.20.2', description='CartoDB Services API Python Library', From 33f40bc945c08c2790e3a385e44568f31bacaed4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 17:38:23 +0200 Subject: [PATCH 229/265] TOMTOM uses 403 instead of 429 for rate limiting That has a great problem when we're dealing with legit 403 status for example deactivated user, forbidden access, etc. I've added a check for the HTTP header `X-Error-Detail-Header` in order to distinguish between legit 403 and 429 error messages Possible values for `X-Error-Detail-Header` in a 403 error: o Service Requires SSL : http is used instead of https (secure) o Invalid Referer : invalid 'Referer' header value is send to https://api.tomtom.com and allowed referer values are configured on specific API key o Account Over Queries Per Second Limit : rate limit exceeded o Account Inactive : incorrect API key/API key no longer valid --- .../python/cartodb_services/cartodb_services/tools/qps.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index 24d110e..9ae3abd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -6,6 +6,9 @@ from exceptions import TimeoutException DEFAULT_RETRY_TIMEOUT = 60 DEFAULT_QUERIES_PER_SECOND = 10 +TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit' +TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header' + def qps_retry(original_function=None, **options): """ Query Per Second retry decorator @@ -46,6 +49,8 @@ class QPSService: response = getattr(e, 'response', None) if response is not None: if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403): + if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER: + raise e self.retry(start_time, attempt_number) elif response.status_code == 429: self.retry(start_time, attempt_number) From 36c42096e420d2bab82f8d6ca38cede2ec77abdb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 31 Oct 2018 13:58:29 +0100 Subject: [PATCH 230/265] Updated NEWS.md --- NEWS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/NEWS.md b/NEWS.md index ca4e3d5..1cc1742 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,15 @@ + +Oct 31th, 2018 +============== +* Version `0.20.2` of the python library + * Added missing provider property to the QPS decorator in other TomTom services + * Now we only retry with the properly header coming from TomTom + +Oct 3rd, 2018 +============== +* Version `0.20.1` of the python library + * Fix QPS manager to retry with 403 status codes coming from TomTom + Sep 13th, 2018 ============== * Version `0.34.0` of the server, and `0.26.0` of the client. From 089d3e3f276bc5c102c884cc486cd5feaf9ece78 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 5 Nov 2018 16:51:17 +0100 Subject: [PATCH 231/265] New version 0.34.1 --- ...db_dataservices_server--0.34.0--0.34.1.sql | 5 + ...db_dataservices_server--0.34.1--0.34.0.sql | 5 + .../cdb_dataservices_server--0.34.1.sql | 3801 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.33.0--0.34.0.sql | 0 ...db_dataservices_server--0.34.0--0.33.0.sql | 0 .../cdb_dataservices_server--0.34.0.sql | 0 7 files changed, 3812 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.34.1.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.33.0--0.34.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.0--0.33.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.0.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql new file mode 100644 index 0000000..642f0da --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql new file mode 100644 index 0000000..7ad4422 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.1.sql new file mode 100644 index 0000000..df6eee6 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.1.sql @@ -0,0 +1,3801 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index ecc5c9b..ff3a969 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.34.0' +default_version = '0.34.1' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.33.0--0.34.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.33.0--0.34.0.sql diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.0--0.33.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.0--0.33.0.sql diff --git a/server/extension/cdb_dataservices_server--0.34.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.0.sql From 2f6d067b4e5e7e05c35276d3941c18dce77535a4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 5 Nov 2018 17:06:52 +0100 Subject: [PATCH 232/265] Parse JSON parameters to avoid problems If we pass JSON as string, with scape characters and so on, we are going to have problems parsing that metrics in ES --- ...db_dataservices_server--0.34.0--0.34.1.sql | 36 ++++++++++++++++++- ...db_dataservices_server--0.34.1--0.34.0.sql | 36 ++++++++++++++++++- .../extension/sql/21_bulk_geocode_street.sql | 3 +- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql index 642f0da..5fd13f5 100644 --- a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql +++ b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql @@ -2,4 +2,38 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql index 7ad4422..8fd4d74 100644 --- a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql +++ b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql @@ -2,4 +2,38 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 173e684..aa8a875 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -9,6 +9,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_poin RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger + import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -20,7 +21,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'searches': searches} + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: From dc47f6f71bf0902440101e08ace3d6626f1ecc6b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Nov 2018 17:51:26 +0100 Subject: [PATCH 233/265] New version 0.34.1 artifacts --- ...db_dataservices_server--0.34.0--0.34.1.sql | 5 + ...db_dataservices_server--0.34.1--0.34.0.sql | 188 + ...ql => cdb_dataservices_server--0.34.1.sql} | 0 .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.33.0--0.34.0.sql | 0 ...db_dataservices_server--0.34.0--0.33.0.sql | 0 .../cdb_dataservices_server--0.34.0.sql | 3800 +++++++++++++++++ 7 files changed, 3994 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql rename server/extension/{cdb_dataservices_server--0.34.0.sql => cdb_dataservices_server--0.34.1.sql} (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.33.0--0.34.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.0--0.33.0.sql (100%) create mode 100644 server/extension/old_versions/cdb_dataservices_server--0.34.0.sql diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql new file mode 100644 index 0000000..642f0da --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql new file mode 100644 index 0000000..248b962 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql @@ -0,0 +1,188 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0.sql rename to server/extension/cdb_dataservices_server--0.34.1.sql diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index ecc5c9b..ff3a969 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.34.0' +default_version = '0.34.1' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.33.0--0.34.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.33.0--0.34.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.33.0--0.34.0.sql diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.0--0.33.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0--0.33.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.0--0.33.0.sql diff --git a/server/extension/old_versions/cdb_dataservices_server--0.34.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.0.sql new file mode 100644 index 0000000..80b1900 --- /dev/null +++ b/server/extension/old_versions/cdb_dataservices_server--0.34.0.sql @@ -0,0 +1,3800 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + service_manager.quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + service_manager.quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; From 87bb8bea68c05b69d45d7700e66a499ebf9bb4a5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Nov 2018 17:59:47 +0100 Subject: [PATCH 234/265] Remove duped metrics for isochrones We are counting the success/empty inside the iteration of the results returned by the third party service. For example, if we ask for 3 isochrones to a provider we count: - 1 for success or empty - If success we add 1 more success for each isochrone in the result - N isolines_generated but it should be - 1 for success or empty - N isolines_generated --- server/extension/sql/80_isolines_helper.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index b0bd598..ade6b65 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -295,10 +295,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) quota_service.increment_success_service_use() quota_service.increment_isolines_service_use(len(result)) @@ -356,10 +354,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) @@ -416,10 +412,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) From 4e1f081952dc2f007752b012e4fe861c0fb128bf Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Nov 2018 18:11:23 +0100 Subject: [PATCH 235/265] Generated 0.34.1 artifact with changes --- server/extension/cdb_dataservices_server--0.34.1.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.1.sql index 80b1900..8197024 100644 --- a/server/extension/cdb_dataservices_server--0.34.1.sql +++ b/server/extension/cdb_dataservices_server--0.34.1.sql @@ -3464,10 +3464,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) quota_service.increment_success_service_use() quota_service.increment_isolines_service_use(len(result)) @@ -3525,10 +3523,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) @@ -3585,10 +3581,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) From 1027d554a507afcb7088c1a11c08d0cf1af4b547 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 19 Nov 2018 18:23:28 +0100 Subject: [PATCH 236/265] Upgrade script from 0.34.0 to 0.34.1 --- ...db_dataservices_server--0.34.0--0.34.1.sql | 179 +++++++++++++++++- 1 file changed, 178 insertions(+), 1 deletion(-) diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql index 642f0da..430d384 100644 --- a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql +++ b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql @@ -2,4 +2,181 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; From 4ae4080bba49036a3a3ffedbb3029e366621dd04 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 22 Nov 2018 15:10:11 +0100 Subject: [PATCH 237/265] Updated NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 1cc1742..9dd993e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +Nov 22th, 2018 +============== +* Version `0.34.1` of the server extension + * Fixed isochrones metrics that increase `success_rows` field for every isochrone generated (#533) Oct 31th, 2018 ============== From 7f4b87f5139ad3ce4b0183f92ddf44fb85b9e500 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Nov 2018 17:39:05 +0100 Subject: [PATCH 238/265] Rename to release --- ....0--0.34.1.sql => cdb_dataservices_server--0.34.1--0.34.2.sql} | 0 ....1--0.34.0.sql => cdb_dataservices_server--0.34.2--0.34.1.sql} | 0 ...ces_server--0.34.1.sql => cdb_dataservices_server--0.34.2.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename server/extension/{cdb_dataservices_server--0.34.0--0.34.1.sql => cdb_dataservices_server--0.34.1--0.34.2.sql} (100%) rename server/extension/{cdb_dataservices_server--0.34.1--0.34.0.sql => cdb_dataservices_server--0.34.2--0.34.1.sql} (100%) rename server/extension/{cdb_dataservices_server--0.34.1.sql => cdb_dataservices_server--0.34.2.sql} (100%) diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.1--0.34.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql rename to server/extension/cdb_dataservices_server--0.34.1--0.34.2.sql diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.2--0.34.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql rename to server/extension/cdb_dataservices_server--0.34.2--0.34.1.sql diff --git a/server/extension/cdb_dataservices_server--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.1.sql rename to server/extension/cdb_dataservices_server--0.34.2.sql From 100319ed0ec9794972f6ab7eff02dca0aff6167c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Nov 2018 17:41:14 +0100 Subject: [PATCH 239/265] Migrate old versions --- server/extension/cdb_dataservices_server.control | 2 +- .../cdb_dataservices_server--0.34.0--0.34.1.sql | 0 .../cdb_dataservices_server--0.34.1--0.34.0.sql | 0 .../{ => old_versions}/cdb_dataservices_server--0.34.1.sql | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.0--0.34.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.1--0.34.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.1.sql (100%) diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index ff3a969..8a1f50f 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.34.1' +default_version = '0.34.2' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.0--0.34.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.0--0.34.1.sql diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.0.sql diff --git a/server/extension/cdb_dataservices_server--0.34.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.1.sql From b2cdb378fbba620fd0feac9d7e9224e632514156 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Nov 2018 17:43:02 +0100 Subject: [PATCH 240/265] Regenerate 0.34.2 artifact --- server/extension/cdb_dataservices_server--0.34.2.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.2.sql b/server/extension/cdb_dataservices_server--0.34.2.sql index df6eee6..453f310 100644 --- a/server/extension/cdb_dataservices_server--0.34.2.sql +++ b/server/extension/cdb_dataservices_server--0.34.2.sql @@ -3465,10 +3465,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) quota_service.increment_success_service_use() quota_service.increment_isolines_service_use(len(result)) @@ -3526,10 +3524,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) @@ -3586,10 +3582,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) service_manager.quota_service.increment_success_service_use() service_manager.quota_service.increment_isolines_service_use(len(result)) From 522591dd631b3d5d1c0e05d5bc9e409bae9e6509 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Nov 2018 17:51:45 +0100 Subject: [PATCH 241/265] Update NEWS.md --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index 9dd993e..9110307 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ + +Nov 27th, 2018 +============== +* Version `0.34.2` of the server extension + * Fixed bulk geocoding log metrics in order to fix problems in ELK due to the JSON not being standard (#530) + Nov 22th, 2018 ============== * Version `0.34.1` of the server extension From 04bbb3284991e7afd71d748cffdeec1b122df2f1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 11:51:59 +0100 Subject: [PATCH 242/265] Added mode type to Tomtom routing functions --- .../cartodb_services/tomtom/routing.py | 10 ++++++---- .../cartodb_services/cartodb_services/tomtom/types.py | 6 ++++++ server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index 71dfa22..bde251c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -11,13 +11,14 @@ from cartodb_services.tools.coordinates import (validate_coordinates, marshall_coordinates) from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry -from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT) +from types import (DEFAULT_PROFILE, DEFAULT_ROUTE_TYPE, VALID_PROFILES, DEFAULT_DEPARTAT) BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/' '{coordinates}' '/json' '?key={apikey}' '&travelMode={travelmode}' + '&routeType={route_type}' '&departAt={departat}' '&computeBestOrder=true') @@ -45,10 +46,11 @@ class TomTomRouting(Traceable): self._logger = logger def _uri(self, coordinates, profile=DEFAULT_PROFILE, - date_time=DEFAULT_DEPARTAT): + date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE): uri = URITemplate(BASEURI).expand(apikey=self._apikey, coordinates=coordinates, travelmode=profile, + route_type=route_type, departat=date_time) return uri @@ -91,13 +93,13 @@ class TomTomRouting(Traceable): @qps_retry(qps=5, provider='tomtom') def directions(self, waypoints, profile=DEFAULT_PROFILE, - date_time=DEFAULT_DEPARTAT): + date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE): self._validate_profile(profile) validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX) coordinates = self._marshall_coordinates(waypoints) - uri = self._uri(coordinates, profile, date_time) + uri = self._uri(coordinates, profile, date_time, route_type) try: response = requests.get(uri) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index 4a7f082..67b1f34 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -24,3 +24,9 @@ TRANSPORT_MODE_TO_TOMTOM = { 'walk': 'pedestrian', 'bicycle': 'bicycle', } + +DEFAULT_ROUTE_TYPE = 'shortest' +MODE_TYPE_TO_TOMTOM = { + 'shortest': 'shortest', + 'fastest': 'fastest' +} diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 02323b8..5263621 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.20.2', + version='0.21.0', description='CartoDB Services API Python Library', From b1bdf2460ef5ffd42ca68cd899be48072d404b73 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 12:09:48 +0100 Subject: [PATCH 243/265] Version 0.35.0 initial files --- server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql | 5 +++++ server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql | 5 +++++ server/extension/cdb_dataservices_server.control | 2 +- .../cdb_dataservices_server--0.34.1--0.34.2.sql | 0 .../cdb_dataservices_server--0.34.2--0.34.1.sql | 0 .../{ => old_versions}/cdb_dataservices_server--0.34.2.sql | 0 6 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.1--0.34.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.2--0.34.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.2.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql new file mode 100644 index 0000000..af4afd1 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql new file mode 100644 index 0000000..f5409df --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql @@ -0,0 +1,5 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.2'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 8a1f50f..5c9341a 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.34.2' +default_version = '0.35.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.1--0.34.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.2.sql diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.34.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.2--0.34.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.2--0.34.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.2--0.34.1.sql diff --git a/server/extension/cdb_dataservices_server--0.34.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.2.sql From 64a0616c978727b793e3a2aeed90957ed43a48bc Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 12:12:02 +0100 Subject: [PATCH 244/265] Added options and units parameters to the routing functions --- server/extension/sql/100_routing_helper.sql | 15 ++++++++---- .../sql/105_route_between_points.sql | 24 +++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index b638f16..9625420 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -8,7 +8,9 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], - mode TEXT) + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxRouting @@ -69,11 +71,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoi username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], - mode TEXT) + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.tools import ServiceManager from cartodb_services.tomtom import TomTomRouting - from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder @@ -104,8 +108,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ waypoint_coords.append(Coordinate(lon,lat)) profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + route_type = DEFAULT_ROUTE_TYPE + if 'mode_type' in options: + route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) - resp = client.directions(waypoint_coords, profile) + resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) if resp and resp.shape: shape_linestring = polyline_to_linestring(resp.shape) if shape_linestring: diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 703e0dd..b6d69c7 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -24,16 +24,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$ waypoints = [origin, destination] if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') @@ -63,16 +63,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$ with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) - result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: raise Exception('Requested routing method is not available') From 90fd8587e2b1710b1c1ed9d3a735d81c98229c5a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 12:12:19 +0100 Subject: [PATCH 245/265] Upgrade and downgrade files --- ...db_dataservices_server--0.34.2--0.35.0.sql | 210 +++++++++++++++++- ...db_dataservices_server--0.35.0--0.34.2.sql | 204 ++++++++++++++++- 2 files changed, 412 insertions(+), 2 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql index af4afd1..3fe344b 100644 --- a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql @@ -2,4 +2,212 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + route_type = DEFAULT_ROUTE_TYPE + if 'mode_type' in options: + route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) + + resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql index f5409df..5b5debb 100644 --- a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql +++ b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql @@ -2,4 +2,206 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.2'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT) +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; From 8d2d0ececd28e5e37346aff076967ba6747e2bbb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 12:34:56 +0100 Subject: [PATCH 246/265] SQL server extension artifact for version 0.35.0 --- .../cdb_dataservices_server--0.35.0.sql | 3802 +++++++++++++++++ 1 file changed, 3802 insertions(+) create mode 100644 server/extension/cdb_dataservices_server--0.35.0.sql diff --git a/server/extension/cdb_dataservices_server--0.35.0.sql b/server/extension/cdb_dataservices_server--0.35.0.sql new file mode 100644 index 0000000..bc6bb23 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.35.0.sql @@ -0,0 +1,3802 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + route_type = DEFAULT_ROUTE_TYPE + if 'mode_type' in options: + route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) + + resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; From 850497ef79978193a034df5938a826d0f8379bd7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 13:02:18 +0100 Subject: [PATCH 247/265] Fix some missing arguments in the functions and add drop for the old ones --- .../extension/cdb_dataservices_server--0.34.2--0.35.0.sql | 8 +++++--- .../extension/cdb_dataservices_server--0.35.0--0.34.2.sql | 2 ++ server/extension/cdb_dataservices_server--0.35.0.sql | 6 +++--- server/extension/sql/105_route_between_points.sql | 6 +++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql index 3fe344b..29f46c6 100644 --- a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql @@ -3,6 +3,8 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit -- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text); CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( username TEXT, orgname TEXT, @@ -197,15 +199,15 @@ RETURNS cdb_dataservices_server.simple_route AS $$ with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql index 5b5debb..bfbfae3 100644 --- a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql +++ b/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql @@ -3,6 +3,8 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.2'" to load this file. \quit -- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text); CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( username TEXT, orgname TEXT, diff --git a/server/extension/cdb_dataservices_server--0.35.0.sql b/server/extension/cdb_dataservices_server--0.35.0.sql index bc6bb23..f68498c 100644 --- a/server/extension/cdb_dataservices_server--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.35.0.sql @@ -261,15 +261,15 @@ RETURNS cdb_dataservices_server.simple_route AS $$ with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index b6d69c7..537feb7 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -63,15 +63,15 @@ RETURNS cdb_dataservices_server.simple_route AS $$ with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): if user_routing_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.mapbox_provider: - mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] elif user_routing_config.tomtom_provider: - tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] else: From 0ec1e051be3a24df76c8f7114e0042431020d062 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 14:15:20 +0100 Subject: [PATCH 248/265] Include options parsing functions --- .../cdb_dataservices_server--0.34.2--0.35.0.sql | 10 ++++++++-- .../cdb_dataservices_server--0.35.0.sql | 10 ++++++++-- server/extension/sql/100_routing_helper.sql | 10 ++++++++-- .../cartodb_services/tomtom/routing.py | 11 ++++++++++- .../cartodb_services/tomtom/types.py | 16 ++++++++++------ .../cartodb_services/tools/normalize.py | 6 ++++++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql index 29f46c6..5ae8a23 100644 --- a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql @@ -18,6 +18,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder import cartodb_services @@ -46,6 +47,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ waypoint_coords.append(Coordinate(lon,lat)) profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + plpy.warning('Mapbox provider doesnt support route type parameter') resp = client.directions(waypoint_coords, profile) if resp and resp.shape: @@ -81,6 +85,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder import cartodb_services @@ -110,8 +115,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) route_type = DEFAULT_ROUTE_TYPE - if 'mode_type' in options: - route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type']) resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) if resp and resp.shape: diff --git a/server/extension/cdb_dataservices_server--0.35.0.sql b/server/extension/cdb_dataservices_server--0.35.0.sql index f68498c..4314dbc 100644 --- a/server/extension/cdb_dataservices_server--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.35.0.sql @@ -20,6 +20,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder import cartodb_services @@ -48,6 +49,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ waypoint_coords.append(Coordinate(lon,lat)) profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + plpy.warning('Mapbox provider doesnt support route type parameter') resp = client.directions(waypoint_coords, profile) if resp and resp.shape: @@ -83,6 +87,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder import cartodb_services @@ -112,8 +117,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) route_type = DEFAULT_ROUTE_TYPE - if 'mode_type' in options: - route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type']) resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) if resp and resp.shape: diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index 9625420..e5f18ad 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -17,6 +17,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder import cartodb_services @@ -45,6 +46,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ waypoint_coords.append(Coordinate(lon,lat)) profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + plpy.warning('Mapbox provider doesnt support route type parameter') resp = client.directions(waypoint_coords, profile) if resp and resp.shape: @@ -80,6 +84,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM from cartodb_services.tools import Coordinate from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder import cartodb_services @@ -109,8 +114,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$ profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) route_type = DEFAULT_ROUTE_TYPE - if 'mode_type' in options: - route_type = MODE_TYPE_TO_TOMTOM.get(options['mode_type']) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type']) resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) if resp and resp.shape: diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index bde251c..a863e25 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -11,7 +11,7 @@ from cartodb_services.tools.coordinates import (validate_coordinates, marshall_coordinates) from cartodb_services.tools.exceptions import ServiceException from cartodb_services.tools.qps import qps_retry -from types import (DEFAULT_PROFILE, DEFAULT_ROUTE_TYPE, VALID_PROFILES, DEFAULT_DEPARTAT) +from types import (DEFAULT_PROFILE, DEFAULT_ROUTE_TYPE, VALID_PROFILES, VALID_ROUTE_TYPE, DEFAULT_DEPARTAT) BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/' '{coordinates}' @@ -62,6 +62,14 @@ class TomTomRouting(Traceable): valid_profiles=', '.join( [x for x in VALID_PROFILES]))) + def _validate_route_type(self, route_type): + if route_type not in VALID_ROUTE_TYPE: + raise ValueError('{route_type} is not a valid route type. ' + 'Valid route types are: {valid_route_types}'.format( + route_type=route_type, + valid_route_types=', '.join( + [x for x in VALID_ROUTE_TYPE]))) + def _marshall_coordinates(self, coordinates): return ':'.join(['{lat},{lon}'.format(lat=coordinate.latitude, lon=coordinate.longitude) @@ -95,6 +103,7 @@ class TomTomRouting(Traceable): def directions(self, waypoints, profile=DEFAULT_PROFILE, date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE): self._validate_profile(profile) + self._validate_route_type(route_type) validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX) coordinates = self._marshall_coordinates(waypoints) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index 67b1f34..71ea26f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -6,12 +6,16 @@ PROFILE_DRIVING = 'car' PROFILE_CYCLING = 'bicycle' PROFILE_WALKING = 'pedestrian' DEFAULT_PROFILE = PROFILE_DRIVING +ROUTE_TYPE_FAST = 'fastest' +ROUTE_TYPE_SHORT = 'shortest' DEFAULT_DEPARTAT = 'now' VALID_PROFILES = [PROFILE_DRIVING, PROFILE_CYCLING, PROFILE_WALKING] +VALID_ROUTE_TYPE = [ROUTE_TYPE_SHORT, + ROUTE_TYPE_FAST] MAX_SPEEDS = { PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed @@ -20,13 +24,13 @@ MAX_SPEEDS = { } TRANSPORT_MODE_TO_TOMTOM = { - 'car': 'car', - 'walk': 'pedestrian', - 'bicycle': 'bicycle', + 'car': PROFILE_DRIVING, + 'walk': PROFILE_WALKING, + 'bicycle': PROFILE_CYCLING, } -DEFAULT_ROUTE_TYPE = 'shortest' +DEFAULT_ROUTE_TYPE = ROUTE_TYPE_SHORT MODE_TYPE_TO_TOMTOM = { - 'shortest': 'shortest', - 'fastest': 'fastest' + 'shortest': ROUTE_TYPE_SHORT, + 'fastest': ROUTE_TYPE_SHORT } diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py index d95f293..dac8066 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py @@ -1,3 +1,9 @@ +from itertools import chain + def normalize(str_input): return str_input.replace('"', '"') \ .replace(';', ',') + +def options_to_dict(options): + options_list = list(chain(*[option.split('=') for option in options])) + return dict(zip(options_list[::2],options_list[1::2])) From 98a4ed81c549ad5941d8f97f9e9cb5f0d9f3e2e6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 16:16:22 +0100 Subject: [PATCH 249/265] Add test for route type verification --- server/lib/python/cartodb_services/test/test_tomtomrouting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/lib/python/cartodb_services/test/test_tomtomrouting.py b/server/lib/python/cartodb_services/test/test_tomtomrouting.py index fab18c4..b4af624 100644 --- a/server/lib/python/cartodb_services/test/test_tomtomrouting.py +++ b/server/lib/python/cartodb_services/test/test_tomtomrouting.py @@ -16,6 +16,7 @@ INVALID_WAYPOINTS_MAX = [Coordinate(13.42936, 52.50931) for x in range(0, NUM_WAYPOINTS_MAX + 2)] VALID_PROFILE = DEFAULT_PROFILE INVALID_PROFILE = 'invalid_profile' +INVALID_ROUTE_TYPE = 'invalid_route_type' class TomTomRoutingTestCase(unittest.TestCase): @@ -43,6 +44,9 @@ class TomTomRoutingTestCase(unittest.TestCase): with self.assertRaises(ServiceException): invalid_routing.directions(VALID_WAYPOINTS, VALID_PROFILE) + def test_invalid_route_type(self): + with self.assertRaises(ValueError): + self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=INVALID_ROUTE_TYPE) def test_valid_request(self): route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE) From a919b876648c63d673631bdc7fa4148a15eba2f5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Nov 2018 16:22:34 +0100 Subject: [PATCH 250/265] Include test for valid request with route type --- .../lib/python/cartodb_services/test/test_tomtomrouting.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/lib/python/cartodb_services/test/test_tomtomrouting.py b/server/lib/python/cartodb_services/test/test_tomtomrouting.py index b4af624..a100b76 100644 --- a/server/lib/python/cartodb_services/test/test_tomtomrouting.py +++ b/server/lib/python/cartodb_services/test/test_tomtomrouting.py @@ -15,6 +15,7 @@ INVALID_WAYPOINTS_MIN = [Coordinate(13.42936, 52.50931)] INVALID_WAYPOINTS_MAX = [Coordinate(13.42936, 52.50931) for x in range(0, NUM_WAYPOINTS_MAX + 2)] VALID_PROFILE = DEFAULT_PROFILE +VALID_ROUTE_TYPE = 'fastest' INVALID_PROFILE = 'invalid_profile' INVALID_ROUTE_TYPE = 'invalid_route_type' @@ -54,3 +55,9 @@ class TomTomRoutingTestCase(unittest.TestCase): assert route.shape assert route.length assert route.duration + + route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=VALID_ROUTE_TYPE) + + assert route.shape + assert route.length + assert route.duration From eb4638a3267a647ec23178110af4109ebc05c8db Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 30 Nov 2018 11:24:12 +0100 Subject: [PATCH 251/265] CR fixes --- .../python/cartodb_services/cartodb_services/tomtom/types.py | 2 +- .../cartodb_services/cartodb_services/tools/normalize.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index 71ea26f..e161a74 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -32,5 +32,5 @@ TRANSPORT_MODE_TO_TOMTOM = { DEFAULT_ROUTE_TYPE = ROUTE_TYPE_SHORT MODE_TYPE_TO_TOMTOM = { 'shortest': ROUTE_TYPE_SHORT, - 'fastest': ROUTE_TYPE_SHORT + 'fastest': ROUTE_TYPE_FAST } diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py index dac8066..6c73ca4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py @@ -5,5 +5,4 @@ def normalize(str_input): .replace(';', ',') def options_to_dict(options): - options_list = list(chain(*[option.split('=') for option in options])) - return dict(zip(options_list[::2],options_list[1::2])) + return dict(option.split("=") for option in options) From 6b6fbef586c8ef15c8ad9040b100341fdba9104e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 30 Nov 2018 11:31:46 +0100 Subject: [PATCH 252/265] Remove unused import --- .../python/cartodb_services/cartodb_services/tools/normalize.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py index 6c73ca4..5888100 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/normalize.py @@ -1,5 +1,3 @@ -from itertools import chain - def normalize(str_input): return str_input.replace('"', '"') \ .replace(';', ',') From c28ca4861ac75e897a35adfdac0fadb5d66c6081 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 30 Nov 2018 12:24:47 +0100 Subject: [PATCH 253/265] Update NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 9110307..8bd6b78 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +Nov 30th, 2018 +============== +* Version `0.35.0` of the server extension + * Now the `mode_type` is working the providers that support it (#536) Nov 27th, 2018 ============== From a8140e53c3f3efa67201f8cb958b43678f7a5ff1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 30 Nov 2018 12:30:02 +0100 Subject: [PATCH 254/265] Update routing mode type doc --- doc/routing_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/routing_functions.md b/doc/routing_functions.md index 17188cd..c563e76 100644 --- a/doc/routing_functions.md +++ b/doc/routing_functions.md @@ -81,4 +81,4 @@ The optional value parameters must be passed using the format: `option=value`. N Name | Type | Description | Accepted values --- | --- | --- | --- -`mode_type` | `text` | Type of route calculation | `shortest` (this option only applies to the car transport mode) +`mode_type` | `text` | Type of route calculation | `shortest` or `fastest` (this option only applies to the car transport mode) From 84276b4cd320199326276fcd6f5c2241fd28c041 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 13:04:16 +0100 Subject: [PATCH 255/265] Do not crash when matchType is empty --- .../cartodb_services/cartodb_services/here/bulk_geocoder.py | 6 +++--- .../cartodb_services/cartodb_services/here/geocoder.py | 4 ++-- server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 772787c..0da38fa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -8,7 +8,7 @@ from collections import namedtuple from requests.adapters import HTTPAdapter from cartodb_services import StreetPointBulkGeocoder from cartodb_services.here import HereMapsGeocoder -from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response +from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED from cartodb_services.metrics import Traceable from cartodb_services.tools.exceptions import ServiceException @@ -137,8 +137,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): reader = csv.DictReader(root_zip.open(name), delimiter='|') for row in reader: if row['SeqNumber'] == '1': # First per requested data - precision = self.PRECISION_BY_MATCH_TYPE[ - row.get('matchType', 'pointAddress')] + precision = self.PRECISION_BY_MATCH_TYPE.get( + row.get('matchType'), PRECISION_INTERPOLATED) match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None) results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']], diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index cbf2576..ae7a369 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -143,8 +143,8 @@ class HereMapsGeocoder(Traceable): def _extract_metadata_from_result(self, result): # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses - precision = self.PRECISION_BY_MATCH_TYPE[ - result.get('MatchType', 'pointAddress')] + precision = self.PRECISION_BY_MATCH_TYPE.get( + result.get('MatchType'), PRECISION_INTERPOLATED) match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None) return geocoder_metadata( result['Relevance'], diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 5263621..8985b8b 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.21.0', + version='0.21.1', description='CartoDB Services API Python Library', From e2c0a63803a7733443226a404fd0da647925a83e Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 17:19:13 +0100 Subject: [PATCH 256/265] Use Point for geocoding results --- client/sql/16_custom_types.sql | 2 +- client/sql/21_bulk_geocoding_functions.sql | 2 +- server/extension/sql/21_bulk_geocode_street.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 15dc782..bb13ee7 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -6,7 +6,7 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( CREATE TYPE cdb_dataservices_client.geocoding AS ( cartodb_id integer, - the_geom geometry(Multipolygon,4326), + the_geom geometry(Point,4326), metadata jsonb ); diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 723ead3..84168fd 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -58,7 +58,7 @@ BEGIN temp_table_name := 'bulk_geocode_street_' || md5(random()::text); EXECUTE format('CREATE TEMPORARY TABLE %s ' || - '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', temp_table_name); select diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index aa8a875..f350e9b 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -1,7 +1,7 @@ -- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? CREATE TYPE cdb_dataservices_server.geocoding AS ( cartodb_id integer, - the_geom geometry(Multipolygon,4326), + the_geom geometry(Point,4326), metadata jsonb ); From 370272f6c482355268224b1a0af1c030cc5b9acf Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 17:21:56 +0100 Subject: [PATCH 257/265] Version bumps --- NEWS.md | 5 +++++ client/carto-package.json | 2 +- client/cdb_dataservices_client.control | 2 +- server/extension/cdb_dataservices_server.control | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8bd6b78..2175715 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +Dec 27th, 2018 +============== +* Version `0.35.1` of the server extension and `0.26.1` of the client + * Now the bulk geocoder returns `Point` geometries instead of MultiPolygons (#538) + Nov 30th, 2018 ============== * Version `0.35.0` of the server extension diff --git a/client/carto-package.json b/client/carto-package.json index df769d7..42ab664 100644 --- a/client/carto-package.json +++ b/client/carto-package.json @@ -7,7 +7,7 @@ "carto_postgresql_ext": "^0.23.0" }, "works_with": { - "dataservices-api-server-extension": "^0.32.0" + "dataservices-api-server-extension": "^0.35.1" } } } diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 31a8d5d..3f363b5 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.26.0' +default_version = '0.26.1' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 5c9341a..9facc8b 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.35.0' +default_version = '0.35.1' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server From 0dfd1a9caa417c4da1601680032302d6d312af75 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 17:28:56 +0100 Subject: [PATCH 258/265] Update/downgrade scripts --- ...db_dataservices_client--0.26.0--0.26.1.sql | 9 + ...db_dataservices_client--0.26.1--0.26.0.sql | 9 + client/cdb_dataservices_client--0.26.1.sql | 5606 +++++++++++++++++ ...db_dataservices_client--0.25.0--0.26.0.sql | 0 ...db_dataservices_client--0.26.0--0.25.0.sql | 0 .../cdb_dataservices_client--0.26.0.sql | 0 ...db_dataservices_server--0.35.0--0.35.1.sql | 6 + ...db_dataservices_server--0.35.1--0.35.0.sql | 6 + .../cdb_dataservices_server--0.35.1.sql | 3808 +++++++++++ ...db_dataservices_server--0.34.2--0.35.0.sql | 0 ...db_dataservices_server--0.35.0--0.34.2.sql | 0 .../cdb_dataservices_server--0.35.0.sql | 0 12 files changed, 9444 insertions(+) create mode 100644 client/cdb_dataservices_client--0.26.0--0.26.1.sql create mode 100644 client/cdb_dataservices_client--0.26.1--0.26.0.sql create mode 100644 client/cdb_dataservices_client--0.26.1.sql rename client/{ => old_versions}/cdb_dataservices_client--0.25.0--0.26.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.26.0--0.25.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.26.0.sql (100%) create mode 100644 server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.35.1.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.34.2--0.35.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.35.0--0.34.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.35.0.sql (100%) diff --git a/client/cdb_dataservices_client--0.26.0--0.26.1.sql b/client/cdb_dataservices_client--0.26.0--0.26.1.sql new file mode 100644 index 0000000..614fd5e --- /dev/null +++ b/client/cdb_dataservices_client--0.26.0--0.26.1.sql @@ -0,0 +1,9 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.1'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(Point,4326); diff --git a/client/cdb_dataservices_client--0.26.1--0.26.0.sql b/client/cdb_dataservices_client--0.26.1--0.26.0.sql new file mode 100644 index 0000000..ae2143b --- /dev/null +++ b/client/cdb_dataservices_client--0.26.1--0.26.0.sql @@ -0,0 +1,9 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.0'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +-- HERE goes your code to upgrade/downgrade +ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(MultiPolygon,4326); diff --git a/client/cdb_dataservices_client--0.26.1.sql b/client/cdb_dataservices_client--0.26.1.sql new file mode 100644 index 0000000..94b5fe7 --- /dev/null +++ b/client/cdb_dataservices_client--0.26.1.sql @@ -0,0 +1,5606 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE;-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text, + apikey_permissions json +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + apikey_config json; + is_organization boolean; + organization_name text DEFAULT NULL; +BEGIN + SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; + + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + END IF; + result.username = apikey_config->>'username'; + result.organization_name = organization_name; + result.apikey_permissions = apikey_config->'permissions'; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Point,4326), + metadata jsonb +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- For the OBS_Meta functions +CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC +); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params) + ) + ) + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + # Create a new table with the required columns + plpy.execute('CREATE TABLE "{schema}".{table_name} ( ' + 'cartodb_id int, the_geom geometry, {columns_with_types} ' + ');' + .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute( + "SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params))) + + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [ + colnames_arr[i] + + ' ' + + coltypes_arr[i] for i in range( + 0, + len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + aliased_colname_list = ','.join( + ['result.' + name for name in colnames_arr]) + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute( + "SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, " + "{schema}::text, {dbname}::text, {table_name}::text);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + user_db_role=plpy.quote_literal(user_db_role), + schema=plpy.quote_literal(user_schema), + dbname=plpy.quote_literal(dbname), + table_name=plpy.quote_literal(table_name))) + + if ds_fdw_metadata[0]["schemaname"]: + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + else: + raise Exception('Error connecting dataset via FDW') + + # Create a new table with the required columns + plpy.execute( + 'INSERT INTO "{schema}".{analysis_table_name} ' + 'SELECT ut.cartodb_id, ut.the_geom, {colname_list} ' + 'FROM "{schema}".{table_name} ut ' + 'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, ' + '{function_name}::text, {params}::json) ' + 'AS result ({columns_with_types}, cartodb_id int) ' + 'ON result.cartodb_id = ut.cartodb_id;' .format( + schema=user_schema, + analysis_table_name=output_table_name, + colname_list=aliased_colname_list, + table_name=table_name, + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params), + columns_with_types=columns_with_types)) + + # Wipe user FDW data from the server + wiped = plpy.execute( + "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, " + "{server_table_name}::text, {fdw_server}::text)" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + fdw_server=plpy.quote_literal(server_name))) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text +)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata( + username text, + orgname text, + function_name text, + params json +) RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData( + username text, + orgname text, + table_schema text, + table_name text, + function_name text, + params json +) RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( + username text, + orgname text, + table_schema text, + table_name text, + server_name text +) RETURNS boolean AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, orgname, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, orgname, ip_address); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, orgname, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_with_waypoints (username, orgname, waypoints, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundary (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundaryid (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundarybyid (username, orgname, geometry_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasure (username, orgname, geom, measure_id, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasurebyid (username, orgname, geom_ref, measure_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomvals, params, merge); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json) +RETURNS TABLE(id text, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomrefs, params); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeta (username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_metadatavalidation (username, orgname, geom_extent, geom_type, params, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getcategory (username, orgname, geom, category_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, orgname, geom, name, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensuscategory (username, orgname, geom, name, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getpopulation (username, orgname, geom, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_search (username, orgname, search_term, relevant_boundary); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailableboundaries (username, orgname, geom, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_dumpversion (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablenumerators (username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, section_tags text[] DEFAULT ARRAY[]::TEXT[], subsection_tags text[] DEFAULT ARRAY[]::TEXT[], other_tags text[] DEFAULT ARRAY[]::TEXT[], ids text[] DEFAULT ARRAY[]::TEXT[], name text DEFAULT NULL, denom_id text DEFAULT '', geom_id text DEFAULT '', timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._obs_getnumerators (username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabledenominators (username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, timespan text DEFAULT NULL, number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablegeometries (username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabletimespans (username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_legacybuildermetadata (username, orgname, aggregate_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info (username text, orgname text) +RETURNS SETOF service_quota_info AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) +RETURNS BOOLEAN AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_enough_quota (username, orgname, service, input_size); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck( + source_query text, + parameters json +) RETURNS boolean AS $$ +DECLARE + errors text[]; + validator_errors text[]; + validator_error text; + valid boolean; + geoms record; +BEGIN + errors := (ARRAY[])::TEXT[]; + FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type, + bool_and(st_isvalid(the_geom)) as valid, + avg(st_npoints(the_geom)) as avg_vertex, + ST_SetSRID(ST_Extent(the_geom), 4326) as extent, + count(*)::INT as numgeoms + FROM (%s) as _source + GROUP BY ST_GeometryType(the_geom)', source_query) + LOOP + IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN + errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type)); + END IF; + + IF geoms.valid IS FALSE THEN + errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them'); + END IF; + + -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here + -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813 + IF geoms.avg_vertex > 1000 THEN + errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them'); + END IF; + + -- OBS specific part + EXECUTE 'SELECT valid, errors + FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)' + INTO valid, validator_errors + USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms; + IF valid is FALSE THEN + FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors + LOOP + errors := array_append(errors, validator_error); + END LOOP; + END IF; + END LOOP; + + IF CARDINALITY(errors) > 0 THEN + RAISE EXCEPTION '%', errors; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_with_waypoints(waypoints geometry(Point, 4326)[], mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom Geometry, category_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom Geometry, name text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom Geometry, name text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text ) TO publicuser; + + + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0--0.26.0.sql b/client/old_versions/cdb_dataservices_client--0.25.0--0.26.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.25.0--0.26.0.sql rename to client/old_versions/cdb_dataservices_client--0.25.0--0.26.0.sql diff --git a/client/cdb_dataservices_client--0.26.0--0.25.0.sql b/client/old_versions/cdb_dataservices_client--0.26.0--0.25.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.26.0--0.25.0.sql rename to client/old_versions/cdb_dataservices_client--0.26.0--0.25.0.sql diff --git a/client/cdb_dataservices_client--0.26.0.sql b/client/old_versions/cdb_dataservices_client--0.26.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.26.0.sql rename to client/old_versions/cdb_dataservices_client--0.26.0.sql diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql b/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql new file mode 100644 index 0000000..240ad8c --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql @@ -0,0 +1,6 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(Point,4326); diff --git a/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql b/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql new file mode 100644 index 0000000..e888347 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql @@ -0,0 +1,6 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(MultiPolygon,4326); diff --git a/server/extension/cdb_dataservices_server--0.35.1.sql b/server/extension/cdb_dataservices_server--0.35.1.sql new file mode 100644 index 0000000..080e7c5 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.35.1.sql @@ -0,0 +1,3808 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + plpy.warning('Mapbox provider doesnt support route type parameter') + + resp = client.directions(waypoint_coords, profile) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate Mapbox routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomRouting + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.tools.normalize import options_to_dict + from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if not waypoints or len(waypoints) < 2: + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + if len(waypoints) > 25: + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + route_type = DEFAULT_ROUTE_TYPE + options_dict = options_to_dict(options) + if 'mode_type' in options_dict: + route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type']) + + resp = client.directions(waypoint_coords, profile=profile, route_type=route_type) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + service_manager.quota_service.increment_success_service_use() + return [shape_linestring, resp.length, int(round(resp.duration))] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + else: + service_manager.quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate TomTom routing') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_point', user_routing_config, logger, params): + waypoints = [origin, destination] + + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units} + + with metrics('cdb_route_with_waypoints', user_routing_config, logger, params): + if user_routing_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + elif user_routing_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] + else: + raise Exception('Requested routing method is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger, params): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_MetadataValidation(geometry_extent, geometry_type, params, target_geoms); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_MetadataValidation( + username TEXT, + orgname TEXT, + geometry_extent Geometry(Geometry, 4326), + geometry_type text, + params JSON, + target_geoms INTEGER DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_metadatavalidation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_MetadataValidation($1, $2, $3, $4, $5, $6);", ["text", "text", "Geometry (Geometry, 4326)", "text", "json", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_extent, geometry_type, params, target_geoms]) + if result: + return result + else: + return [] + except BaseException as e: + import sys + logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_MetadataValidation') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '') +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory._OBS_GetNumerators(bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL, + number_geometries INTEGER DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL SECURITY DEFINER STABLE PARALLEL SAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL SECURITY DEFINER VOLATILE PARALLEL UNSAFE ; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_type' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql STABLE PARALLEL RESTRICTED; +-- Geocodes a street address given a searchtext and a state and/or country + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country} + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapbox_geocoder: + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.tomtom_geocoder: + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.tools import QuotaExceededException + from cartodb_services.here import HereMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager, QuotaExceededException + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + + try: + service_manager.assert_within_limits() + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.mapbox import MapboxGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using Mapbox') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from iso3166 import countries + from cartodb_services.tools import ServiceManager, QuotaExceededException + from cartodb_services.tomtom import TomTomGeocoder + from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + + try: + service_manager.assert_within_limits() + geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + country_iso3166 = None + if country: + country_iso3 = country_to_iso3(country) + if country_iso3: + country_iso3166 = countries.get(country_iso3).alpha2.lower() + + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3166) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using TomTom') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Point,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.google import GoogleMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import LegacyServiceManager + from cartodb_services.here import HereMapsBulkGeocoder + + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder + from cartodb_services.tomtom import TomTomBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD) + geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services import run_street_point_geocoder + from cartodb_services.tools import ServiceManager + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder + from cartodb_services.mapbox import MapboxBulkGeocoder + from cartodb_services.tools import Logger + import cartodb_services + cartodb_services.init(plpy, GD) + + logger_config = GD["logger_config"] + logger = Logger(logger_config) + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches) +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'country_name': country_name} + + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'admin1_name': admin1_name} + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + import spiexceptions + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + try: + street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except spiexceptions.ExternalRoutineException as e: + import sys + logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname}) + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'city_name': city_name, 'admin1_name': admin1_name, 'country_name': country_name} + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger, params): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'code': code} + + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger, params): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text) +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text) +RETURNS Geometry AS $$ + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon(username, orgname, code::text, country); +$$ LANGUAGE SQL STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + params = {'username': username, 'orgname': orgname, 'ip': ip} + + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = tomtom_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.tomtom import TomTomIsolines + from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_TOMTOM.get(mode) + + resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get TomTom isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isodistance', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options} + + with metrics('cdb_isochrone', user_isolines_config, logger, params): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_provider: + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.tomtom_provider: + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- mapbox isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +-- tomtom isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.34.2--0.35.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.34.2--0.35.0.sql diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.35.0--0.34.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.35.0--0.34.2.sql diff --git a/server/extension/cdb_dataservices_server--0.35.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.35.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.35.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.35.0.sql From ac991a05ab104365874dfa9418439fee259f9fce Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 17:36:09 +0100 Subject: [PATCH 259/265] ALTER TYPE syntax fixes --- client/cdb_dataservices_client--0.26.0--0.26.1.sql | 2 +- client/cdb_dataservices_client--0.26.1--0.26.0.sql | 2 +- server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql | 2 +- server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/cdb_dataservices_client--0.26.0--0.26.1.sql b/client/cdb_dataservices_client--0.26.0--0.26.1.sql index 614fd5e..1a0e442 100644 --- a/client/cdb_dataservices_client--0.26.0--0.26.1.sql +++ b/client/cdb_dataservices_client--0.26.0--0.26.1.sql @@ -6,4 +6,4 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(Point,4326); +ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(Point,4326); diff --git a/client/cdb_dataservices_client--0.26.1--0.26.0.sql b/client/cdb_dataservices_client--0.26.1--0.26.0.sql index ae2143b..ab0f3bb 100644 --- a/client/cdb_dataservices_client--0.26.1--0.26.0.sql +++ b/client/cdb_dataservices_client--0.26.1--0.26.0.sql @@ -6,4 +6,4 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(MultiPolygon,4326); +ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(MultiPolygon,4326); diff --git a/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql b/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql index 240ad8c..8aacbd7 100644 --- a/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql +++ b/server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql @@ -3,4 +3,4 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(Point,4326); +ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(Point,4326); diff --git a/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql b/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql index e888347..2fdbc6f 100644 --- a/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql +++ b/server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql @@ -3,4 +3,4 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom SET TYPE geometry(MultiPolygon,4326); +ALTER TYPE cdb_dataservices_server.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(MultiPolygon,4326); From ab13f3a803ecce0fca1db9304563c353348d3bc9 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 24 Dec 2018 17:55:18 +0100 Subject: [PATCH 260/265] Also have to upgrade the geocoding function type --- ...db_dataservices_client--0.26.0--0.26.1.sql | 90 +++++++++++++++++++ ...db_dataservices_client--0.26.1--0.26.0.sql | 90 +++++++++++++++++++ 2 files changed, 180 insertions(+) diff --git a/client/cdb_dataservices_client--0.26.0--0.26.1.sql b/client/cdb_dataservices_client--0.26.0--0.26.1.sql index 1a0e442..06ce6cb 100644 --- a/client/cdb_dataservices_client--0.26.0--0.26.1.sql +++ b/client/cdb_dataservices_client--0.26.0--0.26.1.sql @@ -7,3 +7,93 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(Point,4326); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; diff --git a/client/cdb_dataservices_client--0.26.1--0.26.0.sql b/client/cdb_dataservices_client--0.26.1--0.26.0.sql index ab0f3bb..748cc68 100644 --- a/client/cdb_dataservices_client--0.26.1--0.26.0.sql +++ b/client/cdb_dataservices_client--0.26.1--0.26.0.sql @@ -7,3 +7,93 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(MultiPolygon,4326); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; From 246ec135dbeaa3f9a52717fdac50a4ab040ce22b Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Mon, 31 Dec 2018 11:02:26 +0100 Subject: [PATCH 261/265] Update NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2175715..44565a4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -Dec 27th, 2018 +Dec 31st, 2018 ============== * Version `0.35.1` of the server extension and `0.26.1` of the client * Now the bulk geocoder returns `Point` geometries instead of MultiPolygons (#538) From 29e636f1153d8563e4dae7cd61add27ca09ba12b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 19 Feb 2019 12:24:06 +0100 Subject: [PATCH 262/265] Test for bulk geocoding w/ soft_limit enabled --- .../21_bulk_geocoding_functions_test.out | 21 +++++++++++++++++++ .../sql/21_bulk_geocoding_functions_test.sql | 19 ++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index beebcab..01935ef 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -9,6 +9,13 @@ CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_si RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; +ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked; +CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +BEGIN + RAISE NOTICE 'called with this searches: %', searches; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- No permissions granted -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); @@ -42,6 +49,18 @@ ERROR: Remaining quota: 0. Estimated cost: 1 -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Remaining quota: 0. Estimated cost: 1 +-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota) +-- it is able to proceed with the bulk geocode +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT TRUE; +$$ LANGUAGE SQL; +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +NOTICE: called with this searches: [{"id": 1, "city": "", "state": "", "address": "Valladolid, Spain", "country": ""}] + cdb_bulk_geocode_street_point +------------------------------- +(0 rows) + -- Remove permissions SELECT CDB_Conf_RemoveConf('api_keys_postgres'); cdb_conf_removeconf @@ -51,5 +70,7 @@ SELECT CDB_Conf_RemoveConf('api_keys_postgres'); DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; +DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; +ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index 5b899c8..9457594 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -12,6 +12,14 @@ RETURNS BOOLEAN as $$ SELECT FALSE; $$ LANGUAGE SQL; +ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked; +CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +BEGIN + RAISE NOTICE 'called with this searches: %', searches; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + -- No permissions granted -- Test bulk size not mandatory (it will get the optimal) SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); @@ -32,12 +40,21 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota) +-- it is able to proceed with the bulk geocode +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT TRUE; +$$ LANGUAGE SQL; +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); + -- Remove permissions SELECT CDB_Conf_RemoveConf('api_keys_postgres'); DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; +DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; - +ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point; From f63d73b9d78071cceeafe583ebb104a2a718bb71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 19 Feb 2019 12:27:29 +0100 Subject: [PATCH 263/265] Fix the bulk geocoding when soft_limit is enabled Fix the case where there's not enough quota to geocode a table but the soft_limit is set to true. The function `cdb_enough_quota` accounts for the `soft_limit` flag, as well as for the remaining quota and the rows to be geocoded. --- client/sql/21_bulk_geocoding_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 84168fd..38a0ded 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -49,7 +49,7 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF remaining_quota < query_row_count THEN + IF NOT enough_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; From dfaf0d524529ce75c1b722cef1f44e9d035bd958 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 25 Feb 2019 16:32:01 +0100 Subject: [PATCH 264/265] Prevent use of quota for bogus configurations This is a stop-gap measure for the hypothetical case of a service configuration that yields NULL quota values. --- client/sql/21_bulk_geocoding_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 38a0ded..6e7828a 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -49,7 +49,7 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF NOT enough_quota THEN + IF enough_quota IS NULL OR NOT enough_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; From 850706790153bb38ca650d3310ea7de7d8d0ab94 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 25 Feb 2019 15:02:15 +0100 Subject: [PATCH 265/265] Release 0.26.2 client extension --- NEWS.md | 5 + ...db_dataservices_client--0.26.1--0.26.2.sql | 96 + ...db_dataservices_client--0.26.2--0.26.1.sql | 96 + client/cdb_dataservices_client--0.26.2.sql | 5606 +++++++++++++++++ client/cdb_dataservices_client.control | 2 +- ...db_dataservices_client--0.26.0--0.26.1.sql | 0 ...db_dataservices_client--0.26.1--0.26.0.sql | 0 .../cdb_dataservices_client--0.26.1.sql | 0 8 files changed, 5804 insertions(+), 1 deletion(-) create mode 100644 client/cdb_dataservices_client--0.26.1--0.26.2.sql create mode 100644 client/cdb_dataservices_client--0.26.2--0.26.1.sql create mode 100644 client/cdb_dataservices_client--0.26.2.sql rename client/{ => old_versions}/cdb_dataservices_client--0.26.0--0.26.1.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.26.1--0.26.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.26.1.sql (100%) diff --git a/NEWS.md b/NEWS.md index 44565a4..4755034 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +Feb 25th, 2019 +============== +* Version `0.26.2` of the client extension + * Fixed bulk geocoding soft limit (#543) + Dec 31st, 2018 ============== * Version `0.35.1` of the server extension and `0.26.1` of the client diff --git a/client/cdb_dataservices_client--0.26.1--0.26.2.sql b/client/cdb_dataservices_client--0.26.1--0.26.2.sql new file mode 100644 index 0000000..5abaa37 --- /dev/null +++ b/client/cdb_dataservices_client--0.26.1--0.26.2.sql @@ -0,0 +1,96 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.2'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NULL OR NOT enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; diff --git a/client/cdb_dataservices_client--0.26.2--0.26.1.sql b/client/cdb_dataservices_client--0.26.2--0.26.1.sql new file mode 100644 index 0000000..371a229 --- /dev/null +++ b/client/cdb_dataservices_client--0.26.2--0.26.1.sql @@ -0,0 +1,96 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.1'" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF remaining_quota < query_row_count THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; diff --git a/client/cdb_dataservices_client--0.26.2.sql b/client/cdb_dataservices_client--0.26.2.sql new file mode 100644 index 0000000..4b387c7 --- /dev/null +++ b/client/cdb_dataservices_client--0.26.2.sql @@ -0,0 +1,5606 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit + +-- Make sure we have a sane search path to create/update the extension +SET search_path = "$user",cartodb,public,cdb_dataservices_client; +-- Taken from https://wiki.postgresql.org/wiki/Count_estimate +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS +$func$ +DECLARE + rec record; + ROWS INTEGER; +BEGIN + FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP + ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); + EXIT WHEN ROWS IS NOT NULL; + END LOOP; + + RETURN ROWS; +END +$func$ LANGUAGE plpgsql; + +-- Taken from https://stackoverflow.com/a/48013356/351721 +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ + SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); +$f$ LANGUAGE sql IMMUTABLE;-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text, + apikey_permissions json +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + apikey_config json; + is_organization boolean; + organization_name text DEFAULT NULL; +BEGIN + SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config; + + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + END IF; + result.username = apikey_config->>'username'; + result.organization_name = organization_name; + result.apikey_permissions = apikey_config->'permissions'; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Point,4326), + metadata jsonb +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- For the OBS_Meta functions +CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb); + +CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC +); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundary (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundaryid (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundarybyid (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_metadatavalidation (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getcategory (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensuscategory (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_search (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailableboundaries (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablenumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getnumerators (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabledenominators (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailablegeometries (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getavailabletimespans (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_legacybuildermetadata (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure( + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure( + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username, + orgname, + user_db_role, + user_schema, + dbname, + table_name, + output_table_name, + params + ) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params) + ) + ) + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + # Create a new table with the required columns + plpy.execute('CREATE TABLE "{schema}".{table_name} ( ' + 'cartodb_id int, the_geom geometry, {columns_with_types} ' + ');' + .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text, + output_table_name text, + params json +) RETURNS boolean AS $$ + function_name = 'OBS_GetMeasure' + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute( + "SELECT colnames, coltypes " + "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params))) + + if ds_return_metadata[0]["colnames"]: + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + else: + raise Exception('Error retrieving OBS_GetMeasure metadata') + + # Prepare column and type strings required in the SQL queries + columns_with_types_arr = [ + colnames_arr[i] + + ' ' + + coltypes_arr[i] for i in range( + 0, + len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + aliased_colname_list = ','.join( + ['result.' + name for name in colnames_arr]) + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute( + "SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, " + "{schema}::text, {dbname}::text, {table_name}::text);" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + user_db_role=plpy.quote_literal(user_db_role), + schema=plpy.quote_literal(user_schema), + dbname=plpy.quote_literal(dbname), + table_name=plpy.quote_literal(table_name))) + + if ds_fdw_metadata[0]["schemaname"]: + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + else: + raise Exception('Error connecting dataset via FDW') + + # Create a new table with the required columns + plpy.execute( + 'INSERT INTO "{schema}".{analysis_table_name} ' + 'SELECT ut.cartodb_id, ut.the_geom, {colname_list} ' + 'FROM "{schema}".{table_name} ut ' + 'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, ' + '{function_name}::text, {params}::json) ' + 'AS result ({columns_with_types}, cartodb_id int) ' + 'ON result.cartodb_id = ut.cartodb_id;' .format( + schema=user_schema, + analysis_table_name=output_table_name, + colname_list=aliased_colname_list, + table_name=table_name, + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), + params=plpy.quote_literal(params), + columns_with_types=columns_with_types)) + + # Wipe user FDW data from the server + wiped = plpy.execute( + "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, " + "{server_table_name}::text, {fdw_server}::text)" .format( + username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), + server_schema=plpy.quote_literal(server_schema), + server_table_name=plpy.quote_literal(server_table_name), + fdw_server=plpy.quote_literal(server_name))) + + return True +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable( + username text, + orgname text, + user_db_role text, + user_schema text, + dbname text, + table_name text +)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_ConnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata( + username text, + orgname text, + function_name text, + params json +) RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_GetReturnMetadata; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData( + username text, + orgname text, + table_schema text, + table_name text, + function_name text, + params json +) RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable( + username text, + orgname text, + table_schema text, + table_name text, + server_name text +) RETURNS boolean AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + TARGET cdb_dataservices_server._DST_DisconnectUserTable; +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, + street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + query_row_count integer; + enough_quota boolean; + remaining_quota integer; + max_batch_size integer; + + cartodb_id_batch integer; + batches_n integer; + DEFAULT_BATCH_SIZE CONSTANT numeric := 100; + MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; + + temp_table_name text; + username text; + orgname text; + apikey_permissions json; +BEGIN + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size + INTO remaining_quota, max_batch_size + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi + WHERE service = 'hires_geocoder'; + RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; + + IF batch_size IS NULL THEN + batch_size := max_batch_size; + ELSIF batch_size > max_batch_size THEN + RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1; + END IF; + + IF batch_size > MAX_SAFE_BATCH_SIZE THEN + batch_size := MAX_SAFE_BATCH_SIZE; + END IF; + + EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query) + INTO query_row_count, batches_n; + + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', + query_row_count, query, country_column, state_column, city_column, street_column; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NULL OR NOT enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; + END IF; + + RAISE DEBUG 'batches_n: %', batches_n; + + temp_table_name := 'bulk_geocode_street_' || md5(random()::text); + + EXECUTE format('CREATE TEMPORARY TABLE %s ' || + '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)', + temp_table_name); + + select + coalesce(street_column, ''''''), coalesce(city_column, ''''''), + coalesce(state_column, ''''''), coalesce(country_column, '''''') + into street_column, city_column, state_column, country_column; + + IF batches_n > 0 THEN + FOR cartodb_id_batch in 0..(batches_n - 1) + LOOP + EXECUTE format( + 'WITH geocoding_data as (' || + ' SELECT ' || + ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || + ' floor((row_number() over () - 1)::float/$1) as batch' || + ' FROM (%s) _x' || + ') ' || + 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || + 'FROM geocoding_data ' || + 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) + USING batch_size, cartodb_id_batch; + + END LOOP; + END IF; + + RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe (admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe (city_name text ,admin1_name text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code text ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe (postal_code double precision ,country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN + RAISE EXCEPTION 'Geocoding permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'isolines' THEN + RAISE EXCEPTION 'Isolines permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe (origin geometry(Point, 4326) ,destination geometry(Point, 4326) ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe (waypoints geometry(Point, 4326)[] ,mode text ,options text[] DEFAULT ARRAY[]::text[] ,units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'routing' THEN + RAISE EXCEPTION 'Routing permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT * FROM cdb_dataservices_client._cdb_route_with_waypoints(username, orgname, waypoints, mode, options, units) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT '2009 - 2013'::text ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geom geometry(Geometry, 4326) ,time_span text DEFAULT NULL ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geom geometry(Geometry, 4326) ,geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundary(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundaryid(username, orgname, geom, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe (geometry_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getboundarybyid(username, orgname, geometry_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geom geometry(Geometry, 4326) ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geom geometry(Geometry, 4326) ,radius numeric ,boundary_id text ,time_span text DEFAULT NULL ,overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe (geom Geometry ,measure_id text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe (geom_ref text ,measure_id text ,boundary_id text ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json) +RETURNS TABLE(id text, data json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe (geom_extent Geometry(Geometry, 4326) ,geom_type text ,params json ,target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_metadatavalidation(username, orgname, geom_extent, geom_type, params, target_geoms); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe (geom Geometry ,category_id text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geom Geometry ,name text ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geom Geometry ,name text ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getuscensuscategory(username, orgname, geom, name, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe (geom Geometry ,normalize text DEFAULT NULL ,boundary_id text DEFAULT NULL ,time_span text DEFAULT NULL) +RETURNS numeric AS $$ +DECLARE + ret numeric; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search_exception_safe (search_term text ,relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_search(username, orgname, search_term, relevant_boundary); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geom Geometry ,timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailableboundaries(username, orgname, geom, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe () +RETURNS text AS $$ +DECLARE + ret text; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablenumerators(username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,section_tags text[] DEFAULT ARRAY[]::TEXT[] ,subsection_tags text[] DEFAULT ARRAY[]::TEXT[] ,other_tags text[] DEFAULT ARRAY[]::TEXT[] ,ids text[] DEFAULT ARRAY[]::TEXT[] ,name text DEFAULT NULL ,denom_id text DEFAULT '' ,geom_id text DEFAULT '' ,timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client.__obs_getnumerators(username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,geom_id text DEFAULT NULL ,timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabledenominators(username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,timespan text DEFAULT NULL ,number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailablegeometries(username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe (bounds geometry(Geometry, 4326) DEFAULT NULL ,filter_tags text[] DEFAULT NULL ,numer_id text DEFAULT NULL ,denom_id text DEFAULT NULL ,geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_getavailabletimespans(username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'observatory' THEN + RAISE EXCEPTION 'Data Observatory permission denied'; + END IF; + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._obs_legacybuildermetadata(username, orgname, aggregate_type); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe () +RETURNS SETOF service_quota_info AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN AS $$ +DECLARE + ret BOOLEAN; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_enough_quota(username, orgname, service, input_size) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text) +RETURNS json AS $$ +DECLARE + ret json; + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + apikey_permissions json; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + + + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret; + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + RETURN ret; + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) +RETURNS void AS $$ +DECLARE + + + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; + +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, orgname text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, orgname, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, orgname, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, orgname, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, orgname text, postal_code double precision, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, orgname, postal_code, country_name); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, orgname text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, orgname, ip_address); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, orgname, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, orgname, source, mode, range, options); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, orgname, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[], units text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, orgname text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_with_waypoints (username, orgname, waypoints, mode, options, units); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text, geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, orgname, geom, time_span, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) +RETURNS SETOF JSON AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, orgname, geom, geometry_level); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundary (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundaryid (username, orgname, geom, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, orgname text, geometry_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getboundarybyid (username, orgname, geometry_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, orgname, geom, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, orgname, geom, radius, boundary_id, time_span, overlap_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasure (username, orgname, geom, measure_id, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeasurebyid (username, orgname, geom_ref, measure_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomvals geomval[], params json, merge boolean DEFAULT true) +RETURNS TABLE(id int, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomvals, params, merge); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, orgname text, geomrefs text[], params json) +RETURNS TABLE(id text, data json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getdata (username, orgname, geomrefs, params); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, orgname text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getmeta (username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_metadatavalidation (username text, orgname text, geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer DEFAULT NULL) +RETURNS TABLE(valid boolean, errors text[]) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_metadatavalidation (username, orgname, geom_extent, geom_type, params, target_geoms); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getcategory (username, orgname, geom, category_id, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, orgname, geom, name, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, orgname text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getuscensuscategory (username, orgname, geom, name, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text, boundary_id text, time_span text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL) +RETURNS numeric AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_getpopulation (username, orgname, geom, normalize, boundary_id, time_span); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, orgname text, search_term text, relevant_boundary text DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_search (username, orgname, search_term, relevant_boundary); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, orgname text, geom Geometry, timespan text DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailableboundaries (username, orgname, geom, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.obs_dumpversion (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablenumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablenumerators (username, orgname, bounds, filter_tags, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__obs_getnumerators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, section_tags text[] DEFAULT ARRAY[]::TEXT[], subsection_tags text[] DEFAULT ARRAY[]::TEXT[], other_tags text[] DEFAULT ARRAY[]::TEXT[], ids text[] DEFAULT ARRAY[]::TEXT[], name text DEFAULT NULL, denom_id text DEFAULT '', geom_id text DEFAULT '', timespan text DEFAULT '') +RETURNS SETOF cdb_dataservices_client.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server._obs_getnumerators (username, orgname, bounds, section_tags, subsection_tags, other_tags, ids, name, denom_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabledenominators (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, geom_id text DEFAULT NULL, timespan text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabledenominators (username, orgname, bounds, filter_tags, numer_id, geom_id, timespan); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailablegeometries (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, timespan text DEFAULT NULL, number_geometries integer DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailablegeometries (username, orgname, bounds, filter_tags, numer_id, denom_id, timespan, number_geometries); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailabletimespans (username text, orgname text, bounds geometry(Geometry, 4326) DEFAULT NULL, filter_tags text[] DEFAULT NULL, numer_id text DEFAULT NULL, denom_id text DEFAULT NULL, geom_id text DEFAULT NULL) +RETURNS SETOF cdb_dataservices_client.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_getavailabletimespans (username, orgname, bounds, filter_tags, numer_id, denom_id, geom_id); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_legacybuildermetadata (username text, orgname text, aggregate_type text DEFAULT NULL) +RETURNS TABLE(name text, subsection json) AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.obs_legacybuildermetadata (username, orgname, aggregate_type); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info (username text, orgname text) +RETURNS SETOF service_quota_info AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) +RETURNS BOOLEAN AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_enough_quota (username, orgname, service, input_size); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) +RETURNS json AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) +RETURNS void AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck( + source_query text, + parameters json +) RETURNS boolean AS $$ +DECLARE + errors text[]; + validator_errors text[]; + validator_error text; + valid boolean; + geoms record; +BEGIN + errors := (ARRAY[])::TEXT[]; + FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type, + bool_and(st_isvalid(the_geom)) as valid, + avg(st_npoints(the_geom)) as avg_vertex, + ST_SetSRID(ST_Extent(the_geom), 4326) as extent, + count(*)::INT as numgeoms + FROM (%s) as _source + GROUP BY ST_GeometryType(the_geom)', source_query) + LOOP + IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN + errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type)); + END IF; + + IF geoms.valid IS FALSE THEN + errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them'); + END IF; + + -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here + -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813 + IF geoms.avg_vertex > 1000 THEN + errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them'); + END IF; + + -- OBS specific part + EXECUTE 'SELECT valid, errors + FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)' + INTO valid, validator_errors + USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms; + IF valid is FALSE THEN + FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors + LOOP + errors := array_append(errors, validator_error); + END LOOP; + END IF; + END LOOP; + + IF CARDINALITY(errors) > 0 THEN + RAISE EXCEPTION '%', errors; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code double precision, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code double precision, country_name text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_with_waypoints(waypoints geometry(Point, 4326)[], mode text, options text[], units text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom geometry(Geometry, 4326), time_span text, geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom geometry(Geometry, 4326), geometry_level text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom geometry(Geometry, 4326), boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom Geometry, category_id text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom Geometry, name text, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom Geometry, name text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text ) TO publicuser; + + + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 3f363b5..d474529 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.26.1' +default_version = '0.26.2' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/cdb_dataservices_client--0.26.0--0.26.1.sql b/client/old_versions/cdb_dataservices_client--0.26.0--0.26.1.sql similarity index 100% rename from client/cdb_dataservices_client--0.26.0--0.26.1.sql rename to client/old_versions/cdb_dataservices_client--0.26.0--0.26.1.sql diff --git a/client/cdb_dataservices_client--0.26.1--0.26.0.sql b/client/old_versions/cdb_dataservices_client--0.26.1--0.26.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.26.1--0.26.0.sql rename to client/old_versions/cdb_dataservices_client--0.26.1--0.26.0.sql diff --git a/client/cdb_dataservices_client--0.26.1.sql b/client/old_versions/cdb_dataservices_client--0.26.1.sql similarity index 100% rename from client/cdb_dataservices_client--0.26.1.sql rename to client/old_versions/cdb_dataservices_client--0.26.1.sql