diff --git a/NEWS.md b/NEWS.md index 78f590f..7491aed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,21 @@ +March 13th, 2017 +================ +* Version `0.14.1` of the python library: + * Clean up code that reads from non zero padded date keys #206 + +March 8th, 2017 +=============== +* Version 0.22.0 of the server and version 0.14.0 of the python library + * New optional configuration parameters for external services can be provided through `cdb_conf`: + - In `heremaps_conf`, under `geocoder.service`: `json_url`, `connect_timeout`, `read_timeout`, `max_retries`, `gen` + - In `heremaps_conf`, under `isolines.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries`, `isoline_path` + - In `mapzen_conf`, under `geocoder.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries` + - In `mapzen_conf`, under `routing.service`: `base_url`, `connect_timeout`, `read_timeout` + - In `mapzen_conf`, under `matrix.service`: `one_to_many_url`, `connect_timeout`, `read_timeout` + - In `mapzen_conf`, under `isochrones.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries` + * Strictly speaking, version 0.14.0 of the python library is not compatible with 0.13.0, but the changes are made on method signatures with default values that were not used from the PG extension. + * Improvements to the Mapzen geocoder client, that should yield better results overall. See #342 + February 2st, 2017 =================== * Version 0.21.0 of the server and version 0.15.0 of the client diff --git a/README.md b/README.md index c597812..9c6588d 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ Steps to deploy a new Data Services API version : ### Local install instructions -- install data services geocoder extension +- install data services geocoder extension ``` git clone https://github.com/CartoDB/data-services.git cd data-services/geocoder/extension sudo make install ``` - -- install observatory extension + +- install observatory extension ``` git clone https://github.com/CartoDB/observatory-extension.git @@ -40,7 +40,7 @@ Steps to deploy a new Data Services API version : ``` - install server and client extensions - + ``` # in dataservices-api repo root path: cd client && sudo make install @@ -51,7 +51,7 @@ Steps to deploy a new Data Services API version : ``` # in dataservices-api repo root path: - cd server/lib/python/cartodb_services && sudo pip install . --upgrade + cd server/lib/python/cartodb_services && sudo pip install . --upgrade ``` - install extensions in user database @@ -71,11 +71,11 @@ Steps to deploy a new Data Services API version : # If sentinel is used: SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - + # If sentinel is not used SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); - + SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}'); SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}'); @@ -88,15 +88,183 @@ Steps to deploy a new Data Services API version : - configure the user DB: - ```sql - -- Point to the dataservices server DB (you can use a specific database for the server or your same user's): - SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); +### Server configuration - SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); - ``` +Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`. -- configure the search path in order to be able to execute the functions without using the schema: +#### Redis configuration - ``` - ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; - ``` +If sentinel is used: + +```sql +SELECT CDB_Conf_SetConf( + 'redis_metadata_config', + '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}' +); +SELECT CDB_Conf_SetConf( + 'redis_metrics_config', + '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}' +); +``` + +If sentinel is not used: + +```sql +SELECT CDB_Conf_SetConf( + 'redis_metadata_config', + '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}' +); +SELECT CDB_Conf_SetConf( + 'redis_metrics_config', + '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}' +); +``` + +#### Users/Organizations + +```sql +SELECT CDB_Conf_SetConf( + 'user_config', + '{"is_organization": false, "entity_name": ""}' +); +``` + +#### HERE configuration + +```sql +SELECT CDB_Conf_SetConf( + 'heremaps_conf', + '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}' +); +``` + +#### Mapzen configuration + +```sql +SELECT CDB_Conf_SetConf( + 'mapzen_conf', + '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}' +); +``` + +#### Data Observatory + +```sql +SELECT CDB_Conf_SetConf( + 'data_observatory_conf', + '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}' +); +``` + +#### Logger + +```sql +SELECT CDB_Conf_SetConf( + 'logger_conf', + '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}' +); +``` + +#### Environment + +The execution environment (development/staging/production) affects rollbar messages and other details. +The production environment is used by default. + +```sql +SELECT CDB_Conf_SetConf( + 'server_conf', + '{"environment": "[development|staging|production]"}' +); +``` +### Server optional configuration + +External services (Mapzen, Here) can have optional configuration, which is only needed for using non-standard services, such as on-premise installations. We can add the service parameters to an existing configuration like this: + +``` +# Here geocoder +SELECT CDB_Conf_SetConf( +'heremaps_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('heremaps_conf')), + '{geocoder, service}', + '{"json_url":"https://geocoder.api.here.com/6.2/geocode.json","gen":9,"read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Here isolines +SELECT CDB_Conf_SetConf( +'heremaps_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('heremaps_conf')), + '{isolines, service}', + '{"base_url":"https://isoline.route.api.here.com","isoline_path":"/routing/7.2/calculateisoline.json","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen geocoder +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{geocoder, service}', + '{"base_url":"https://search.mapzen.com/v1/search","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen isochrones +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{isochrones, service}', + '{"base_url":"https://matrix.mapzen.com/isochrone","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen isolines (matrix service) +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{matrix, service}', + '{"base_url":"https://matrix.mapzen.com/one_to_many","read_timeout":60,"connect_timeout":10}' +)::json +); + +# Mapzen routing +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{routing, service}', + '{"base_url":"https://valhalla.mapzen.com/route","read_timeout":60,"connect_timeout":10}' +)::json +); +``` +### User database configuration + +User (client) databases need also some configuration so that the client extension can access the server: +#### Users/Organizations + +```sql +SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); +``` + +#### Dataservices server + +The `geocoder_server_config` (the name is not accurate for historical reasons) entry points +to the dataservices server DB (you can use a specific database for the server or your same user's): + +```sql +SELECT CDB_Conf_SetConf( + 'geocoder_server_config', + '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}' +); +``` +#### Search path + +The search path must be configured in order to be able to execute the functions without using the schema: + +```sql +ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; +``` diff --git a/server/extension/Makefile b/server/extension/Makefile index bb98dc2..090210f 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -38,7 +38,7 @@ all: $(DATA) .PHONY: release release: $(EXTENSION).control $(SOURCES_DATA) test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0 - mv *.sql old_versions + git mv *.sql old_versions $(SED) $(REPLACEMENTS) $(EXTENSION).control cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql $(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql diff --git a/server/extension/cdb_dataservices_server--0.21.0--0.22.0.sql b/server/extension/cdb_dataservices_server--0.21.0--0.22.0.sql new file mode 100644 index 0000000..b441abb --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.21.0--0.22.0.sql @@ -0,0 +1,292 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.22.0'" to load this file. \quit + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger = Logger(logger_config) + + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.22.0--0.21.0.sql b/server/extension/cdb_dataservices_server--0.22.0--0.21.0.sql new file mode 100644 index 0000000..849690a --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.22.0--0.21.0.sql @@ -0,0 +1,292 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.21.0'" to load this file. \quit + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger = Logger(logger_config) + + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.22.0.sql b/server/extension/cdb_dataservices_server--0.22.0.sql new file mode 100644 index 0000000..77255d0 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.22.0.sql @@ -0,0 +1,2900 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_config.connection_str +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + empty_results = len(geomvals) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomvals)) + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomvals)) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + empty_results = len(geomrefs) - len(result) + if empty_results > 0: + quota_service.increment_empty_service_use(empty_results) + if result: + quota_service.increment_success_service_use(len(result)) + return result + else: + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use(len(geomrefs)) + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use(len(geomrefs)) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('obs_getmeta', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + return result[0]['meta'] + else: + return None + except BaseException as e: + import sys + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_search', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getavailableboundaries', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundary', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundaryid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundarybyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getboundariesbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbygeometry', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getpointsbypointandradius', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy; + +-- We could create a super type for the common data like id, name and so on but we need to parse inside the functions because the -- the return data tha comes from OBS is a TABLE() with them +CREATE TYPE cdb_dataservices_server.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableNumerators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_numerator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableNumerators(bounds, filter_tags, denom_id, geom_id, timespan); +$$ LANGUAGE plproxy; + +CREATE TYPE cdb_dataservices_server.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableDenominators( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan); +$$ LANGUAGE plproxy; + +CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan); +$$ LANGUAGE plproxy; + +CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans( + username TEXT, + orgname TEXT, + bounds geometry(Geometry, 4326) DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL) +RETURNS SETOF cdb_dataservices_server.obs_meta_timespan AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableTimespans(bounds, filter_tags, numer_id, denom_id, geom_id); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_LegacyBuilderMetadata( + username TEXT, + orgname TEXT, + aggregate_type TEXT DEFAULT NULL) +RETURNS TABLE(name TEXT, subsection JSON) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_LegacyBuilderMetadata(aggregate_type); +$$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL STABLE SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_quota_info') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger = Logger(logger_config) + + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen'))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country_name: + country_iso3 = country_to_iso3(country_name) + coordinates = geocoder.geocode(searchtext=city_name, city=None, + state_province=admin1_name, + country=country_iso3, search_type='locality') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode city point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig, metrics + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsochrones + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.mapzen.types import coordinates_to_polygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, + logger, user_isolines_routing_config.mapzen_isochrones_service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + resp = mapzen_isochrones.isochrone(origin, mode, data_range) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + quota_service.increment_success_service_use() + result.append([source, isochrone.duration, result_polygon]) + else: + quota_service.increment_empty_service_use() + result.append([source, isochrone.duration, None]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(result)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isochrones') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isodistance', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index b9e017a..6c2fbbd 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.21.0' +default_version = '0.22.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.20.0--0.21.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.20.0--0.21.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.20.0--0.21.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.20.0--0.21.0.sql diff --git a/server/extension/cdb_dataservices_server--0.21.0--0.20.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.21.0--0.20.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.21.0--0.20.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.21.0--0.20.0.sql diff --git a/server/extension/cdb_dataservices_server--0.21.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.21.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.21.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.21.0.sql diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index efa79bf..681d240 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -31,7 +31,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('You have reached the limit of your quota') try: - client = MapzenRouting(user_routing_config.mapzen_api_key, logger) + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) if not waypoints or len(waypoints) < 2: logger.info("Empty origin or destination") diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index fd85d6d..eaf7dc1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -85,7 +85,7 @@ RETURNS Geometry AS $$ service_manager.check() try: - geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, logger) + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -162,7 +162,7 @@ RETURNS Geometry AS $$ service_manager.check() try: - geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, logger) + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) country_iso3 = None if country: untry_iso3 = country_to_iso3(country) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 029c828..2e1bfb2 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -20,7 +20,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('You have reached the limit of your quota') try: - client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, logger) if source: @@ -81,7 +81,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('You have reached the limit of your quota') try: - client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger) + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) mapzen_isolines = MapzenIsolines(client, logger) if source: @@ -151,7 +151,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ try: mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, - logger) + logger, user_isolines_routing_config.mapzen_isochrones_service_params) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index a3eb805..6cd0a7b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -52,14 +52,17 @@ class HereMapsGeocoder(Traceable): 'strictlanguagemode' ] + ADDRESS_PARAMS - def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS, - gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS): + service_params = service_params or {} self.app_id = app_id self.app_code = app_code self._logger = logger self.maxresults = maxresults - self.gen = gen - self.host = host + self.gen = service_params.get('gen', self.DEFAULT_GEN) + self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) def geocode(self, **kwargs): params = {} @@ -92,9 +95,9 @@ class HereMapsGeocoder(Traceable): request_params.update(params) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self.host, HTTPAdapter(max_retries=self.max_retries)) response = session.get(self.host, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self.connect_timeout, self.read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return json.loads(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 3a04a3a..fcc2e3e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -30,12 +30,17 @@ class HereMapsRoutingIsoline(Traceable): 'quality' ] - def __init__(self, app_id, app_code, logger, - base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_id, app_code, logger, service_params=None): + service_params = service_params or {} self._app_id = app_id self._app_code = app_code self._logger = logger - self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH) + base_url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL) + isoline_path = service_params.get('isoline_path', self.ISOLINE_PATH) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) + self._url = "{0}{1}".format(base_url, isoline_path) def calculate_isodistance(self, source, mode, data_range, options=[]): return self.__calculate_isolines(source, mode, data_range, 'distance', @@ -57,9 +62,9 @@ class HereMapsRoutingIsoline(Traceable): parsed_options) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self.max_retries)) response = requests.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self.connect_timeout, self.read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_isolines_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 1209388..e43ee30 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -17,23 +17,32 @@ class MapzenGeocoder(Traceable): CONNECT_TIMEOUT = 10 MAX_RETRIES = 1 - def __init__(self, app_key, logger, base_url=BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url + self._url = service_params.get('base_url', self.BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) self._logger = logger @qps_retry(qps=20) def geocode(self, searchtext, city=None, state_province=None, country=None, search_type=None): + + # Remove the search_type if its address from the params sent to mapzen + if search_type and search_type.lower() == 'address': + search_type = None + request_params = self._build_requests_parameters(searchtext, city, state_province, country, search_type) try: # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py index f7f15fd..8b17c57 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py @@ -20,10 +20,15 @@ class MapzenIsochrones: "car": "auto" } - def __init__(self, app_key, logger, base_url=BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url self._logger = logger + self._url = service_params.get('base_url', self.BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) + @qps_retry(qps=7) def isochrone(self, locations, costing, ranges): @@ -32,10 +37,10 @@ class MapzenIsochrones: try: # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, - self.READ_TIMEOUT)) + timeout=(self._connect_timeout, + self._read_timeout)) if response.status_code is requests.codes.ok: return self._parse_response(response) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index c10a912..041b382 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -23,9 +23,14 @@ class MatrixClient(Traceable): READ_TIMEOUT = 60 CONNECT_TIMEOUT = 10 - def __init__(self, matrix_key, logger): + def __init__(self, matrix_key, logger, service_params=None): + service_params = service_params or {} self._matrix_key = matrix_key self._logger = logger + self._url = service_params.get('one_to_many_url', self.ONE_TO_MANY_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + """Get distances and times to a set of locations. See https://mapzen.com/documentation/matrix/api-reference/ @@ -44,8 +49,8 @@ class MatrixClient(Traceable): 'costing': costing, 'api_key': self._matrix_key } - response = requests.get(self.ONE_TO_MANY_URL, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + response = requests.get(self._url, params=request_params, + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code != requests.codes.ok: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index 0cb0c80..ac7e909 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -33,10 +33,14 @@ class MapzenRouting(Traceable): METRICS_UNITS = 'kilometers' IMPERIAL_UNITS = 'miles' - def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url self._logger = logger + self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) @qps_retry def calculate_route_point_to_point(self, waypoints, mode, @@ -50,9 +54,9 @@ class MapzenRouting(Traceable): request_params = self.__parse_request_parameters(json_request_params) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d4328a1..e87608d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -154,6 +154,7 @@ class RoutingConfig(ServiceConfig): if not self._routing_provider: self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key + self._mapzen_service_params = self._db_config.mapzen_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -171,6 +172,10 @@ class RoutingConfig(ServiceConfig): def mapzen_api_key(self): return self._mapzen_api_key + @property + def mapzen_service_params(self): + return self._mapzen_service_params + @property def monthly_quota(self): return self._monthly_quota @@ -229,8 +234,11 @@ class IsolinesRoutingConfig(ServiceConfig): if self._isolines_provider == self.HEREMAPS_PROVIDER: self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code + self._heremaps_service_params = db_config.heremaps_isolines_service_params elif self._isolines_provider == self.MAPZEN_PROVIDER: self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key + self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params + self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params @property def service_type(self): @@ -263,10 +271,22 @@ class IsolinesRoutingConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_service_params(self): + return self._heremaps_service_params + @property def mapzen_matrix_api_key(self): return self._mapzen_matrix_api_key + @property + def mapzen_matrix_service_params(self): + return self._mapzen_matrix_service_params + + @property + def mapzen_isochrones_service_params(self): + return self._mapzen_isochrones_service_params + @property def mapzen_provider(self): return self._isolines_provider == self.MAPZEN_PROVIDER @@ -375,6 +395,7 @@ class GeocoderConfig(ServiceConfig): self._heremaps_app_id = db_config.heremaps_geocoder_app_id self._heremaps_app_code = db_config.heremaps_geocoder_app_code self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit + self._heremaps_service_params = db_config.heremaps_geocoder_service_params elif self._geocoder_provider == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] @@ -382,6 +403,7 @@ class GeocoderConfig(ServiceConfig): elif self._geocoder_provider == self.MAPZEN_GEOCODER: self._mapzen_api_key = db_config.mapzen_geocoder_api_key self._cost_per_hit = 0 + self._mapzen_service_params = db_config.mapzen_geocoder_service_params self._rate_limit = self._get_rate_limit('geocoder') @@ -437,10 +459,18 @@ class GeocoderConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_service_params(self): + return self._heremaps_service_params + @property def mapzen_api_key(self): return self._mapzen_api_key + @property + def mapzen_service_params(self): + return self._mapzen_service_params + @property def is_high_resolution(self): return True @@ -457,6 +487,10 @@ class GeocoderConfig(ServiceConfig): def rate_limit(self): return self._rate_limit + @property + def service(self): + return self._service + class ServicesDBConfig: def __init__(self, db_conn, username, orgname): @@ -493,8 +527,10 @@ class ServicesDBConfig: self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ 'geocoder_cost_per_hit'] + self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {}) self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] + self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {}) def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -504,10 +540,14 @@ class ServicesDBConfig: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] + self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {}) + self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {}) self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] + self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {}) self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] + self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') @@ -547,6 +587,10 @@ class ServicesDBConfig: def heremaps_isolines_app_code(self): return self._heremaps_isolines_app_code + @property + def heremaps_isolines_service_params(self): + return self._heremaps_isolines_service_params + @property def heremaps_geocoder_app_id(self): return self._heremaps_geocoder_app_id @@ -559,6 +603,10 @@ class ServicesDBConfig: def heremaps_geocoder_cost_per_hit(self): return self._heremaps_geocoder_cost_per_hit + @property + def heremaps_geocoder_service_params(self): + return self._heremaps_geocoder_service_params + @property def mapzen_matrix_api_key(self): return self._mapzen_matrix_api_key @@ -567,6 +615,14 @@ class ServicesDBConfig: def mapzen_matrix_monthly_quota(self): return self._mapzen_matrix_quota + @property + def mapzen_matrix_service_params(self): + return self._mapzen_matrix_service_params + + @property + def mapzen_isochrones_service_params(self): + return self._mapzen_isochrones_service_params + @property def mapzen_routing_api_key(self): return self._mapzen_routing_api_key @@ -575,6 +631,10 @@ class ServicesDBConfig: def mapzen_routing_monthly_quota(self): return self._mapzen_routing_quota + @property + def mapzen_routing_service_params(self): + return self._mapzen_routing_service_params + @property def mapzen_geocoder_api_key(self): return self._mapzen_geocoder_api_key @@ -583,6 +643,10 @@ class ServicesDBConfig: def mapzen_geocoder_monthly_quota(self): return self._mapzen_geocoder_quota + @property + def mapzen_geocoder_service_params(self): + return self._mapzen_geocoder_service_params + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 82aaf95..a5f64b6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -118,12 +118,9 @@ class UserMetricsService: for date in self.__generate_date_range(date_from, date_to): redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, service, metric, date) - score = self._redis_connection.zscore(redis_prefix, date.day) - aggregated_metric += int(score) if score else 0 zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED) - if str(date.day) != zero_padded_day: - score = self._redis_connection.zscore(redis_prefix, zero_padded_day) - aggregated_metric += int(score) if score else 0 + score = self._redis_connection.zscore(redis_prefix, zero_padded_day) + aggregated_metric += int(score) if score else 0 return aggregated_metric diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 4c5884a..af74e00 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object): log_path, mapzen_api_key, username, - organization): + organization, + service_params): self._geocoding_quota = geocoding_quota self._soft_geocoding_limit = soft_geocoding_limit self._period_end_date = period_end_date @@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object): self._mapzen_api_key = mapzen_api_key self._username = username self._organization = organization + self._service_params = service_params # Kind of generic properties. Note which ones are for actually running the # service and which ones are needed for quota stuff. @@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object): def organization(self): return self._organization + @property + def service_params(self): + return self._service_params + # TODO: for BW compat, remove @property def google_geocoder(self): @@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object): def get(self): mapzen_server_conf = self._server_conf.get('mapzen_conf') mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] + mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {}) geocoding_quota = self._get_quota(mapzen_server_conf) soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' @@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object): log_path, mapzen_api_key, self._username, - self._orgname) + self._orgname, + mapzen_service_params) def _get_quota(self, mapzen_server_conf): geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 5e5be34..d103715 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.13.0', + version='0.14.1', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py index d67a2bf..b61f4e6 100644 --- a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py @@ -13,7 +13,7 @@ class TestMapzenGeocoderUserConfig(TestCase): self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"}, "mapzen_conf": {"geocoder": - {"api_key": "search-xxxxxxx", "monthly_quota": 1500000} + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}} }, "logger_conf": {}}) self._username = 'test_user' self._user_key = "rails:users:{0}".format(self._username) @@ -81,6 +81,14 @@ class TestMapzenGeocoderUserConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00') + def test_config_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.service_params == {"base_url":"http://base"} + class TestMapzenGeocoderOrgConfig(TestCase): def setUp(self): @@ -151,4 +159,12 @@ class TestMapzenGeocoderOrgConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00') self._redis_connection.hset(self._org_key, 'geocoding_quota', '200') - self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') \ No newline at end of file + self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') + + def test_config_default_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.service_params == {} diff --git a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py index c09d75b..daf813d 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py @@ -145,3 +145,17 @@ class HereMapsGeocoderTestCase(unittest.TestCase): searchtext='Calle amor de dios', city='Cordoba', country='España') + + def test_geocode_with_nonstandard_url(self, req_mock): + geocoder = HereMapsGeocoder(None, None, Mock(), { 'json_url': 'http://nonstandard_here_url' }) + req_mock.register_uri('GET', 'http://nonstandard_here_url', text=self.GOOD_RESPONSE) + response = geocoder.geocode( + searchtext='Calle amor de dios', + city='Cordoba', + country='España') + + self.assertEqual(response[0], -5.2794) + self.assertEqual(response[1], 37.70246) + + + diff --git a/server/lib/python/cartodb_services/test/test_heremapsrouting.py b/server/lib/python/cartodb_services/test/test_heremapsrouting.py index 6ce247a..6226572 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsrouting.py +++ b/server/lib/python/cartodb_services/test/test_heremapsrouting.py @@ -212,3 +212,20 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase): parsed_url = urlparse(req_mock.request_history[0].url) url_params = parse_qs(parsed_url.query) self.assertEqual(url_params['destination'][0], 'geo!33.0,1.0') + + def test_isodistance_with_nonstandard_url(self, req_mock): + base_url = 'http://nonstandard_base' + url = "{0}{1}".format(base_url, HereMapsRoutingIsoline.ISOLINE_PATH) + routing = HereMapsRoutingIsoline(None, None, Mock(), { 'base_url': base_url }) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + response = routing.calculate_isodistance('geo!33.0,1.0', 'car', + ['1000', '2000']) + self.assertEqual(len(response), 2) + self.assertEqual(response[0]['range'], 1000) + self.assertEqual(response[1]['range'], 2000) + self.assertEqual(response[0]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9458542', + u'32.9699707,0.9462833']) + self.assertEqual(response[1]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9750366', + u'32.9699707,0.9462833']) diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py index 19d8f42..3a3b1e6 100644 --- a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -109,3 +109,14 @@ class MapzenGeocoderTestCase(unittest.TestCase): self.geocoder.geocode( searchtext='Calle Siempreviva 3, Valladolid', country='ESP') + + def test_geocode_address_with_nonstandard_url(self, req_mock): + nonstandard_url = 'http://nonstandardmapzen' + req_mock.register_uri('GET', nonstandard_url, text=self.GOOD_RESPONSE) + geocoder = MapzenGeocoder('search-XXXXXXX', Mock(), { 'base_url': nonstandard_url }) + response = geocoder.geocode( + searchtext='Calle Siempreviva 3, Valldolid', + country='ESP') + + self.assertEqual(response[0], -4.730928) + self.assertEqual(response[1], 41.669034) diff --git a/server/lib/python/cartodb_services/test/test_mapzenisochrones.py b/server/lib/python/cartodb_services/test/test_mapzenisochrones.py index 6b566c4..184b010 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenisochrones.py +++ b/server/lib/python/cartodb_services/test/test_mapzenisochrones.py @@ -52,3 +52,16 @@ class MapzenIsochronesTestCase(unittest.TestCase): with self.assertRaises(ServiceException): self.mapzen_isochrones.isochrone([-41.484375, 28.993727], 'walk', [300, 900]) + + def test_nonstandard_url(self, req_mock): + url = 'http://serviceurl.com' + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + mapzen_isochrones = MapzenIsochrones('matrix-xxxxx', Mock(), {'base_url': url}) + + response = mapzen_isochrones.isochrone([-41.484375, 28.993727], + 'walk', [300, 900]) + self.assertEqual(len(response), 2) + self.assertEqual(response[0].coordinates, [[-3.702579,40.430893],[-3.702193,40.430122],[-3.702579,40.430893]]) + self.assertEqual(response[0].duration, 15) + self.assertEqual(response[1].coordinates, [[-3.703050,40.424995],[-3.702546,40.424694],[-3.703050,40.424995]]) + self.assertEqual(response[1].duration, 5) diff --git a/server/lib/python/cartodb_services/test/test_mapzenrouting.py b/server/lib/python/cartodb_services/test/test_mapzenrouting.py index 2315146..916558a 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapzenrouting.py @@ -142,3 +142,17 @@ class MapzenRoutingTestCase(unittest.TestCase): self.assertEqual(response.length, 1.261) self.assertEqual(response.duration, 913) self.assertEqual(response.shape, self.GOOD_SHAPE_MULTI) + + def test_nonstandard_url(self, req_mock): + url = 'http://serviceurl.com' + routing = MapzenRouting('api_key', Mock(), {'base_url': url}) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE_SIMPLE) + origin = Coordinate('-120.2', '38.5') + destination = Coordinate('-126.4', '43.2') + waypoints = [origin, destination] + response = routing.calculate_route_point_to_point(waypoints, + 'car') + + self.assertEqual(response.shape, self.GOOD_SHAPE_SIMPLE) + self.assertEqual(response.length, 444.59) + self.assertEqual(response.duration, 16969) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 201a1de..f1b3598 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -90,19 +90,6 @@ class TestUserService(TestCase): self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400) assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400 - @freeze_time("2015-06-01") - def test_should_account_for_wrongly_stored_non_padded_keys(self): - us = self.__build_user_service('test_user') - self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400) - assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400 - - @freeze_time("2015-06-01") - def test_should_sum_amounts_from_both_key_formats(self): - us = self.__build_user_service('test_user') - self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400) - self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300) - assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700 - @freeze_time("2015-06-15") def test_should_not_request_redis_twice_when_unneeded(self): class MockRedisWithCounter(MockRedis):