diff --git a/server/extension/Makefile b/server/extension/Makefile index d764817..9e00c53 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.6.1--0.6.0.sql \ - cdb_dataservices_server--0.6.0--0.6.1.sql + cdb_dataservices_server--0.6.2--0.6.1.sql \ + cdb_dataservices_server--0.6.1--0.6.2.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.6.1--0.6.2.sql b/server/extension/cdb_dataservices_server--0.6.1--0.6.2.sql new file mode 100644 index 0000000..b749c0a --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.1--0.6.2.sql @@ -0,0 +1,102 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.2'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key) + + if not origin or not destination: + plpy.notice("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.6.2--0.6.1.sql b/server/extension/cdb_dataservices_server--0.6.2--0.6.1.sql new file mode 100644 index 0000000..f09abda --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.2--0.6.1.sql @@ -0,0 +1,98 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + if not origin or not destination: + plpy.notice("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.6.2.sql b/server/extension/cdb_dataservices_server--0.6.2.sql new file mode 100644 index 0000000..506c69d --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.2.sql @@ -0,0 +1,1024 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key) + + if not origin or not destination: + plpy.notice("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_point_to_point($1, $2, $3, $4, $5, $6, $7) as route;", ["text", "text", "geometry(Point, 4326)", "geometry(Point, 4326)", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 682ea2a..a276382 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.6.1' +default_version = '0.6.2' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.0--0.6.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.0--0.6.1.sql diff --git a/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.1--0.6.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.1--0.6.0.sql diff --git a/server/extension/cdb_dataservices_server--0.6.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.1.sql similarity index 99% rename from server/extension/cdb_dataservices_server--0.6.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.1.sql index 8c8989a..bd8d743 100644 --- a/server/extension/cdb_dataservices_server--0.6.1.sql +++ b/server/extension/old_versions/cdb_dataservices_server--0.6.1.sql @@ -27,6 +27,8 @@ RETURNS cdb_dataservices_server.simple_route AS $$ user_routing_config = GD["user_routing_config_{0}".format(username)] quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') try: client = MapzenRouting(user_routing_config.mapzen_app_key) @@ -257,6 +259,8 @@ RETURNS Geometry AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') try: geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index 6e561ba..603753f 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -24,9 +24,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ user_routing_config = GD["user_routing_config_{0}".format(username)] quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') try: - client = MapzenRouting(user_routing_config.mapzen_app_key) + client = MapzenRouting(user_routing_config.mapzen_api_key) if not origin or not destination: plpy.notice("Empty origin or destination") diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index ae34242..035aeef 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -89,18 +89,23 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) country_iso3 = None if country: country_iso3 = country_to_iso3(country) - coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) if coordinates: quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index d6951fd..9c217f6 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -25,7 +25,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy (1 row) -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index d86e983..efd55f5 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -12,7 +12,7 @@ CREATE EXTENSION cdb_dataservices_server; SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); -- Mock the varnish invalidation function diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 7731b0f..fd63419 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -10,10 +10,15 @@ class ConfigException(Exception): class ServiceConfig(object): __metaclass__ = abc.ABCMeta - def __init__(self, redis_connection, username, orgname=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): self._redis_connection = redis_connection self._username = username self._orgname = orgname + self._db_config = ServicesDBConfig(db_conn) + if redis_connection: + self._redis_config = ServicesRedisConfig(redis_connection).build(username, orgname) + else: + self._redis_config = None @abc.abstractproperty def service_type(self): @@ -30,24 +35,30 @@ class ServiceConfig(object): class RoutingConfig(ServiceConfig): - ROUTING_CONFIG_KEYS = ['username', 'orgname', 'mapzen_app_key'] - MAPZEN_APP_KEY = 'mapzen_app_key' - USERNAME_KEY = 'username' - ORGNAME_KEY = 'orgname' + PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, db_conn, username, orgname=None): - super(RoutingConfig, self).__init__(redis_connection, username, - orgname) - db_config = ServicesDBConfig(db_conn) - self._mapzen_app_key = db_config.mapzen_routing_app_key + super(RoutingConfig, self).__init__(redis_connection, db_conn, + username, orgname) + self._mapzen_api_key = self._db_config.mapzen_routing_api_key + self._monthly_quota = self._db_config.mapzen_routing_monthly_quota + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property def service_type(self): return 'routing_mapzen' @property - def mapzen_app_key(self): - return self._mapzen_app_key + def mapzen_api_key(self): + return self._mapzen_api_key + + @property + def monthly_quota(self): + return self._monthly_quota + + @property + def period_end_date(self): + return self._period_end_date class IsolinesRoutingConfig(ServiceConfig): @@ -56,43 +67,17 @@ class IsolinesRoutingConfig(ServiceConfig): 'period_end_date', 'username', 'orgname', 'heremaps_isolines_app_id', 'heremaps_isolines_app_code', 'geocoder_type'] - NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id' - NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code' QUOTA_KEY = 'here_isolines_quota' SOFT_LIMIT_KEY = 'soft_here_isolines_limit' - USERNAME_KEY = 'username' - ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' GEOCODER_TYPE_KEY = 'geocoder_type' GOOGLE_GEOCODER = 'google' def __init__(self, redis_connection, db_conn, username, orgname=None): - super(IsolinesRoutingConfig, self).__init__(redis_connection, username, - orgname) - config = self.__get_user_config(username, orgname) - db_config = ServicesDBConfig(db_conn) - filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} - self.__parse_config(filtered_config, db_config) - - def __get_user_config(self, username, orgname): - user_config = self._redis_connection.hgetall( - "rails:users:{0}".format(username)) - if not user_config: - raise ConfigException("""There is no user config available. Please check your configuration.'""") - else: - if orgname: - self.__get_organization_config(orgname, user_config) - - return user_config - - def __get_organization_config(self, orgname, user_config): - org_config = self._redis_connection.hgetall( - "rails:orgs:{0}".format(orgname)) - if not org_config: - raise ConfigException("""There is no organization config available. Please check your configuration.'""") - else: - user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] - user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn, + username, orgname) + filtered_config = {key: self._redis_config[key] for key in self.ROUTING_CONFIG_KEYS if key in self._redis_config.keys()} + self.__parse_config(filtered_config, self._db_config) def __parse_config(self, filtered_config, db_config): self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower() @@ -137,10 +122,10 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): - super(InternalGeocoderConfig, self).__init__(redis_connection, + # For now, internal geocoder doesn't use the redis config + super(InternalGeocoderConfig, self).__init__(None, db_conn, username, orgname) - db_config = ServicesDBConfig(db_conn) - self._log_path = db_config.geocoder_log_path + self._log_path = self._db_config.geocoder_log_path @property def service_type(self): @@ -169,7 +154,7 @@ class GeocoderConfig(ServiceConfig): 'geocoding_quota', 'soft_geocoding_limit', 'geocoder_type', 'period_end_date', 'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code', - 'mapzen_geocoder_app_key', 'username', 'orgname'] + 'mapzen_geocoder_api_key', 'username', 'orgname'] NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id' @@ -178,7 +163,7 @@ class GeocoderConfig(ServiceConfig): GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' MAPZEN_GEOCODER = 'mapzen' - MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_app_key' + MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key' GEOCODER_TYPE = 'geocoder_type' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' @@ -187,12 +172,10 @@ class GeocoderConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, db_conn, username, orgname=None): - super(GeocoderConfig, self).__init__(redis_connection, username, - orgname) - db_config = ServicesDBConfig(db_conn) - config = self.__get_user_config(username, orgname) - filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} - self.__parse_config(filtered_config, db_config) + super(GeocoderConfig, self).__init__(redis_connection, db_conn, + username, orgname) + filtered_config = {key: self._redis_config[key] for key in self.GEOCODER_CONFIG_KEYS if key in self._redis_config.keys()} + self.__parse_config(filtered_config, self._db_config) self.__check_config(filtered_config) def __get_user_config(self, username, orgname): @@ -226,7 +209,7 @@ class GeocoderConfig(ServiceConfig): if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: - if not self.mapzen_app_key: + if not self.mapzen_api_key: raise ConfigException("""Mapzen config is not setted up""") return True @@ -249,7 +232,8 @@ class GeocoderConfig(ServiceConfig): self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] self._cost_per_hit = 0 elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: - self._mapzen_app_key = db_config.mapzen_geocoder_app_key + self._mapzen_api_key = db_config.mapzen_geocoder_api_key + self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota self._cost_per_hit = 0 @property @@ -283,10 +267,10 @@ class GeocoderConfig(ServiceConfig): @property def geocoding_quota(self): - if self.heremaps_geocoder: - return self._geocoding_quota - else: + if self.google_geocoder: return None + else: + return self._geocoding_quota @property def soft_geocoding_limit(self): @@ -305,8 +289,8 @@ class GeocoderConfig(ServiceConfig): return self._heremaps_app_code @property - def mapzen_app_key(self): - return self._mapzen_app_key + def mapzen_api_key(self): + return self._mapzen_api_key @property def is_high_resolution(self): @@ -351,8 +335,10 @@ class ServicesDBConfig: raise ConfigException('Mapzen configuration missing') else: mapzen_conf = json.loads(mapzen_conf_json) - self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] - self._mapzen_geocoder_app_key = mapzen_conf['geocoder_app_key'] + self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] + self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] + self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] + self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] def _get_logger_config(self): logger_conf_json = self._get_conf('logger_conf') @@ -391,13 +377,57 @@ class ServicesDBConfig: return self._heremaps_geocoder_cost_per_hit @property - def mapzen_routing_app_key(self): - return self._mapzen_routing_app_key + def mapzen_routing_api_key(self): + return self._mapzen_routing_api_key @property - def mapzen_geocoder_app_key(self): - return self._mapzen_geocoder_app_key + def mapzen_routing_monthly_quota(self): + return self._mapzen_routing_quota + + @property + def mapzen_geocoder_api_key(self): + return self._mapzen_geocoder_api_key + + @property + def mapzen_geocoder_monthly_quota(self): + return self._mapzen_geocoder_quota @property def geocoder_log_path(self): return self._geocoder_log_path + + +class ServicesRedisConfig: + + GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' + GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' + QUOTA_KEY = 'geocoding_quota' + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_conn): + self._redis_connection = redis_conn + + def build(self, username, password): + return self.__get_user_config(username, password) + + def __get_user_config(self, username, orgname): + user_config = self._redis_connection.hgetall( + "rails:users:{0}".format(username)) + if not user_config: + raise ConfigException("""There is no user config available. Please check your configuration.'""") + else: + if orgname: + self.__get_organization_config(orgname, user_config) + + return user_config + + def __get_organization_config(self, orgname, user_config): + org_config = self._redis_connection.hgetall( + "rails:orgs:{0}".format(orgname)) + if not org_config: + raise ConfigException("""There is no organization config available. Please check your configuration.'""") + else: + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] + user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 7eefd11..238fb4e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -70,6 +70,9 @@ class QuotaChecker: elif re.match('here_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('routing_mapzen', + self._user_service_config.service_type) is not None: + return self.__check_routing_quota() else: return False @@ -100,3 +103,14 @@ class QuotaChecker: return True else: return False + + def __check_routing_quota(self): + user_quota = self._user_service_config.monthly_quota + today = date.today() + service_type = self._user_service_config.service_type + current_used = self._user_service.used_quota(service_type, today) + + if (user_quota > 0 and current_used <= user_quota): + return True + else: + return False diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 782746d..d154e0c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -41,11 +41,12 @@ class UserMetricsService: return current_use def __used_isolines_quota(self, service_type, date): + """ Recover the used quota for the user in the current month """ date_from, date_to = self.__current_billing_cycle() current_use = 0 isolines_generated = self.get_metrics(service_type, - 'isolines_generated', date_from, - date_to) + 'isolines_generated', date_from, + date_to) empty_responses = self.get_metrics(service_type, 'empty_responses', date_from, date_to) @@ -53,12 +54,27 @@ class UserMetricsService: return current_use + def __used_routing_quota(self, service_type, date): + """ Recover the used quota for the user in the current month """ + date_from, date_to = self.__current_billing_cycle() + current_use = 0 + success_responses = self.get_metrics(service_type, + 'success_responses', date_from, + date_to) + empty_responses = self.get_metrics(service_type, + 'empty_responses', date_from, + date_to) + current_use += (success_responses + empty_responses) - def increment_service_use(self, service_type, metric, date=date.today(), amount=1): + return current_use + + def increment_service_use(self, service_type, metric, date=date.today(), + amount=1): """ Increment the services uses in monthly and daily basis""" self.__increment_user_uses(service_type, metric, date, amount) if self._orgname: - self.__increment_organization_uses(service_type, metric, date, amount) + self.__increment_organization_uses(service_type, metric, date, + amount) def get_metrics(self, service, metric, date_from, date_to): aggregated_metric = 0 @@ -83,11 +99,12 @@ class UserMetricsService: service_type, metric, date) self._redis_connection.zincrby(redis_prefix, date.day, amount) - def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date): + def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, + date): yearmonth_key = date.strftime('%Y%m') redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, - service_type, metric, - yearmonth_key) + service_type, metric, + yearmonth_key) return redis_name diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index f3f0f04..562de35 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.4', + version='0.4.5', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 349b043..9347d62 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -46,6 +46,6 @@ def _plpy_execute_side_effect(*args, **kwargs): if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}] + return [{'conf': '{"routing": {"api_key": "valhalla-Z61FWEs", "monthly_quota": 1500000}, "geocoder": {"api_key": "search-d744tp0", "monthly_quota": 1500000}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 9df7f88..8f4414b 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis from cartodb_services.metrics import QuotaService -from cartodb_services.metrics import GeocoderConfig +from cartodb_services.metrics import GeocoderConfig, RoutingConfig from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -18,79 +18,124 @@ class TestQuotaService(TestCase): self.redis_conn = MockRedis() def test_should_return_true_if_user_quota_with_no_use(self): - qs = self.__build_quota_service('test_user') + qs = self.__build_geocoder_quota_service('test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_with_no_use(self): - qs = self.__build_quota_service('test_user', orgname='test_org') + qs = self.__build_geocoder_quota_service('test_user', + orgname='test_org') assert qs.check_user_quota() is True def test_should_return_true_if_user_quota_is_not_completely_used(self): - qs = self.__build_quota_service('test_user') + qs = self.__build_geocoder_quota_service('test_user') test_helper.increment_geocoder_uses(self.redis_conn, 'test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_not_completely_used(self): - qs = self.__build_quota_service('test_user', orgname='test_org') + qs = self.__build_geocoder_quota_service('test_user', + orgname='test_org') test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): - qs = self.__build_quota_service('test_user') + qs = self.__build_geocoder_quota_service('test_user') test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): - qs = self.__build_quota_service('test_user', orgname='test_org') + qs = self.__build_geocoder_quota_service('test_user', + orgname='test_org') test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service('test_user', soft_limit=True) + qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service('test_user', orgname='test_org', - soft_limit=True) + qs = self.__build_geocoder_quota_service('test_user', + orgname='test_org', + soft_limit=True) test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is True def test_should_check_user_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service('test_user', quota=2) + qs = self.__build_geocoder_quota_service('test_user', quota=2) qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=2) assert qs.check_user_quota() is False - month = date.today().strftime('%Y%m') def test_should_check_org_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service('test_user', orgname='test_org', - quota=2) + qs = self.__build_geocoder_quota_service('test_user', quota=2, + orgname='test_org') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=2) assert qs.check_user_quota() is False - month = date.today().strftime('%Y%m') - def __build_quota_service(self, username, quota=100, service='heremaps', - orgname=None, soft_limit=False, - end_date = datetime.today()): + def test_should_check_user_mapzen_geocoder_quota_correctly(self): + qs = self.__build_geocoder_quota_service('test_user', service='mapzen') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_org_mapzen_geocoder_quota_correctly(self): + qs = self.__build_geocoder_quota_service('test_user', orgname='testorg', + service='mapzen') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_user_routing_quota_correctly(self): + qs = self.__build_routing_quota_service('test_user') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_org_routing_quota_correctly(self): + qs = self.__build_routing_quota_service('test_user', orgname='testorg') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def __prepare_quota_service(self, username, quota, service, orgname, + soft_limit, end_date): test_helper.build_redis_user_config(self.redis_conn, username, - quota = quota, service = service, - soft_limit = soft_limit, - end_date = end_date) + quota=quota, service=service, + soft_limit=soft_limit, + end_date=end_date) if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - plpy_mock = test_helper.build_plpy_mock() - geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, - username, orgname) - return QuotaService(geocoder_config, - redis_connection = self.redis_conn) + self._plpy_mock = test_helper.build_plpy_mock() + def __build_geocoder_quota_service(self, username, quota=100, + service='heremaps', orgname=None, + soft_limit=False, + end_date=datetime.today()): + self.__prepare_quota_service(username, quota, service, orgname, + soft_limit, end_date) + geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, + username, orgname) + return QuotaService(geocoder_config, redis_connection=self.redis_conn) + + def __build_routing_quota_service(self, username, quota=100, + service='routing_mapzen', orgname=None, + soft_limit=False, + end_date=datetime.today()): + self.__prepare_quota_service(username, quota, service, orgname, + soft_limit, end_date) + routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, + username, orgname) + return QuotaService(routing_config, redis_connection=self.redis_conn)