From fbd42e3db0784b8c288ee40070d5bd961f72f365 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 Mar 2016 13:03:09 +0100 Subject: [PATCH 01/10] Scaffolding for the 0.5.1 version --- .../extension/cdb_dataservices_server.control | 2 +- server/extension/sql/0.5.1/00_header.sql | 1 + .../sql/0.5.1/100_routing_helper.sql | 55 +++++ .../sql/0.5.1/105_route_point_to_point.sql | 1 + .../extension/sql/0.5.1/10_redis_helper.sql | 1 + .../extension/sql/0.5.1/15_config_helper.sql | 1 + .../extension/sql/0.5.1/20_geocode_street.sql | 84 +++++++ server/extension/sql/0.5.1/30_admin0.sql | 50 ++++ server/extension/sql/0.5.1/40_admin1.sql | 117 ++++++++++ server/extension/sql/0.5.1/50_namedplaces.sql | 164 +++++++++++++ server/extension/sql/0.5.1/60_postalcodes.sql | 219 ++++++++++++++++++ server/extension/sql/0.5.1/70_ips.sql | 61 +++++ .../sql/0.5.1/80_isolines_helper.sql | 54 +++++ server/extension/sql/0.5.1/85_isodistance.sql | 1 + server/extension/sql/0.5.1/90_isochrone.sql | 1 + .../sql/0.5.1/999_geocoder_server_user.sql | 1 + .../test/0.5.0/expected/00_install_test.out | 4 +- .../test/0.5.1/expected/00_install_test.out | 1 + .../test/0.5.1/expected/20_street_test.out | 1 + .../test/0.5.1/expected/30_admin0_test.out | 1 + .../test/0.5.1/expected/40_admin1_test.out | 1 + .../0.5.1/expected/50_namedplaces_test.out | 1 + .../0.5.1/expected/60_postalcodes_test.out | 1 + .../test/0.5.1/expected/70_ips_test.out | 1 + .../0.5.1/expected/85_isodistance_test.out | 1 + .../test/0.5.1/expected/90_isochrone_test.out | 1 + .../expected/95_route_point_to_point_test.out | 1 + .../999_remove_geocoder_api_user_test.out | 1 + .../test/0.5.1/sql/00_install_test.sql | 26 +++ .../test/0.5.1/sql/20_street_test.sql | 1 + .../test/0.5.1/sql/30_admin0_test.sql | 1 + .../test/0.5.1/sql/40_admin1_test.sql | 1 + .../test/0.5.1/sql/50_namedplaces_test.sql | 1 + .../test/0.5.1/sql/60_postalcodes_test.sql | 1 + .../extension/test/0.5.1/sql/70_ips_test.sql | 1 + .../test/0.5.1/sql/85_isodistance_test.sql | 1 + .../test/0.5.1/sql/90_isochrone_test.sql | 1 + .../sql/95_route_point_to_point_test.sql | 1 + .../sql/999_remove_geocoder_api_user_test.sql | 1 + 39 files changed, 861 insertions(+), 3 deletions(-) create mode 120000 server/extension/sql/0.5.1/00_header.sql create mode 100644 server/extension/sql/0.5.1/100_routing_helper.sql create mode 120000 server/extension/sql/0.5.1/105_route_point_to_point.sql create mode 120000 server/extension/sql/0.5.1/10_redis_helper.sql create mode 120000 server/extension/sql/0.5.1/15_config_helper.sql create mode 100644 server/extension/sql/0.5.1/20_geocode_street.sql create mode 100644 server/extension/sql/0.5.1/30_admin0.sql create mode 100644 server/extension/sql/0.5.1/40_admin1.sql create mode 100644 server/extension/sql/0.5.1/50_namedplaces.sql create mode 100644 server/extension/sql/0.5.1/60_postalcodes.sql create mode 100644 server/extension/sql/0.5.1/70_ips.sql create mode 100644 server/extension/sql/0.5.1/80_isolines_helper.sql create mode 120000 server/extension/sql/0.5.1/85_isodistance.sql create mode 120000 server/extension/sql/0.5.1/90_isochrone.sql create mode 120000 server/extension/sql/0.5.1/999_geocoder_server_user.sql create mode 120000 server/extension/test/0.5.1/expected/00_install_test.out create mode 120000 server/extension/test/0.5.1/expected/20_street_test.out create mode 120000 server/extension/test/0.5.1/expected/30_admin0_test.out create mode 120000 server/extension/test/0.5.1/expected/40_admin1_test.out create mode 120000 server/extension/test/0.5.1/expected/50_namedplaces_test.out create mode 120000 server/extension/test/0.5.1/expected/60_postalcodes_test.out create mode 120000 server/extension/test/0.5.1/expected/70_ips_test.out create mode 120000 server/extension/test/0.5.1/expected/85_isodistance_test.out create mode 120000 server/extension/test/0.5.1/expected/90_isochrone_test.out create mode 120000 server/extension/test/0.5.1/expected/95_route_point_to_point_test.out create mode 120000 server/extension/test/0.5.1/expected/999_remove_geocoder_api_user_test.out create mode 100644 server/extension/test/0.5.1/sql/00_install_test.sql create mode 120000 server/extension/test/0.5.1/sql/20_street_test.sql create mode 120000 server/extension/test/0.5.1/sql/30_admin0_test.sql create mode 120000 server/extension/test/0.5.1/sql/40_admin1_test.sql create mode 120000 server/extension/test/0.5.1/sql/50_namedplaces_test.sql create mode 120000 server/extension/test/0.5.1/sql/60_postalcodes_test.sql create mode 120000 server/extension/test/0.5.1/sql/70_ips_test.sql create mode 120000 server/extension/test/0.5.1/sql/85_isodistance_test.sql create mode 120000 server/extension/test/0.5.1/sql/90_isochrone_test.sql create mode 120000 server/extension/test/0.5.1/sql/95_route_point_to_point_test.sql create mode 120000 server/extension/test/0.5.1/sql/999_remove_geocoder_api_user_test.sql diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 677c0e8..bbae55f 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.5.0' +default_version = '0.5.1' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/0.5.1/00_header.sql b/server/extension/sql/0.5.1/00_header.sql new file mode 120000 index 0000000..6d12de2 --- /dev/null +++ b/server/extension/sql/0.5.1/00_header.sql @@ -0,0 +1 @@ +../0.4.0/00_header.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/100_routing_helper.sql b/server/extension/sql/0.5.1/100_routing_helper.sql new file mode 100644 index 0000000..e23a1c5 --- /dev/null +++ b/server/extension/sql/0.5.1/100_routing_helper.sql @@ -0,0 +1,55 @@ +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_geocoder_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/105_route_point_to_point.sql b/server/extension/sql/0.5.1/105_route_point_to_point.sql new file mode 120000 index 0000000..049f5df --- /dev/null +++ b/server/extension/sql/0.5.1/105_route_point_to_point.sql @@ -0,0 +1 @@ +../0.5.0/105_route_point_to_point.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/10_redis_helper.sql b/server/extension/sql/0.5.1/10_redis_helper.sql new file mode 120000 index 0000000..71e8ff6 --- /dev/null +++ b/server/extension/sql/0.5.1/10_redis_helper.sql @@ -0,0 +1 @@ +../0.5.0/10_redis_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/15_config_helper.sql b/server/extension/sql/0.5.1/15_config_helper.sql new file mode 120000 index 0000000..fdce38c --- /dev/null +++ b/server/extension/sql/0.5.1/15_config_helper.sql @@ -0,0 +1 @@ +../0.5.0/15_config_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/20_geocode_street.sql b/server/extension/sql/0.5.1/20_geocode_street.sql new file mode 100644 index 0000000..de77f62 --- /dev/null +++ b/server/extension/sql/0.5.1/20_geocode_street.sql @@ -0,0 +1,84 @@ +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.1/30_admin0.sql b/server/extension/sql/0.5.1/30_admin0.sql new file mode 100644 index 0000000..ec8fe5c --- /dev/null +++ b/server/extension/sql/0.5.1/30_admin0.sql @@ -0,0 +1,50 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.1/40_admin1.sql b/server/extension/sql/0.5.1/40_admin1.sql new file mode 100644 index 0000000..dc8e037 --- /dev/null +++ b/server/extension/sql/0.5.1/40_admin1.sql @@ -0,0 +1,117 @@ +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.5.1/50_namedplaces.sql b/server/extension/sql/0.5.1/50_namedplaces.sql new file mode 100644 index 0000000..80306cb --- /dev/null +++ b/server/extension/sql/0.5.1/50_namedplaces.sql @@ -0,0 +1,164 @@ +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.5.1/60_postalcodes.sql b/server/extension/sql/0.5.1/60_postalcodes.sql new file mode 100644 index 0000000..4ba321b --- /dev/null +++ b/server/extension/sql/0.5.1/60_postalcodes.sql @@ -0,0 +1,219 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.1/70_ips.sql b/server/extension/sql/0.5.1/70_ips.sql new file mode 100644 index 0000000..f76190f --- /dev/null +++ b/server/extension/sql/0.5.1/70_ips.sql @@ -0,0 +1,61 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.1/80_isolines_helper.sql b/server/extension/sql/0.5.1/80_isolines_helper.sql new file mode 100644 index 0000000..eb15963 --- /dev/null +++ b/server/extension/sql/0.5.1/80_isolines_helper.sql @@ -0,0 +1,54 @@ +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/85_isodistance.sql b/server/extension/sql/0.5.1/85_isodistance.sql new file mode 120000 index 0000000..3b2e4dc --- /dev/null +++ b/server/extension/sql/0.5.1/85_isodistance.sql @@ -0,0 +1 @@ +../0.4.0/85_isodistance.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/90_isochrone.sql b/server/extension/sql/0.5.1/90_isochrone.sql new file mode 120000 index 0000000..59276d2 --- /dev/null +++ b/server/extension/sql/0.5.1/90_isochrone.sql @@ -0,0 +1 @@ +../0.4.0/90_isochrone.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/999_geocoder_server_user.sql b/server/extension/sql/0.5.1/999_geocoder_server_user.sql new file mode 120000 index 0000000..916112a --- /dev/null +++ b/server/extension/sql/0.5.1/999_geocoder_server_user.sql @@ -0,0 +1 @@ +../0.5.0/999_geocoder_server_user.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/00_install_test.out b/server/extension/test/0.5.0/expected/00_install_test.out index 2da3255..c61b0a2 100644 --- a/server/extension/test/0.5.0/expected/00_install_test.out +++ b/server/extension/test/0.5.0/expected/00_install_test.out @@ -7,13 +7,13 @@ CREATE EXTENSION cdb_geocoder; -- Install the extension CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db -SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); cdb_conf_setconf ------------------ (1 row) -SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/0.5.1/expected/00_install_test.out b/server/extension/test/0.5.1/expected/00_install_test.out new file mode 120000 index 0000000..9a720f2 --- /dev/null +++ b/server/extension/test/0.5.1/expected/00_install_test.out @@ -0,0 +1 @@ +../../0.5.0/expected/00_install_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/20_street_test.out b/server/extension/test/0.5.1/expected/20_street_test.out new file mode 120000 index 0000000..6081021 --- /dev/null +++ b/server/extension/test/0.5.1/expected/20_street_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/20_street_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/30_admin0_test.out b/server/extension/test/0.5.1/expected/30_admin0_test.out new file mode 120000 index 0000000..9cfed39 --- /dev/null +++ b/server/extension/test/0.5.1/expected/30_admin0_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/30_admin0_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/40_admin1_test.out b/server/extension/test/0.5.1/expected/40_admin1_test.out new file mode 120000 index 0000000..32b2ff2 --- /dev/null +++ b/server/extension/test/0.5.1/expected/40_admin1_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/40_admin1_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/50_namedplaces_test.out b/server/extension/test/0.5.1/expected/50_namedplaces_test.out new file mode 120000 index 0000000..ebc8d68 --- /dev/null +++ b/server/extension/test/0.5.1/expected/50_namedplaces_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/50_namedplaces_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/60_postalcodes_test.out b/server/extension/test/0.5.1/expected/60_postalcodes_test.out new file mode 120000 index 0000000..147ebf6 --- /dev/null +++ b/server/extension/test/0.5.1/expected/60_postalcodes_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/60_postalcodes_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/70_ips_test.out b/server/extension/test/0.5.1/expected/70_ips_test.out new file mode 120000 index 0000000..9b61caa --- /dev/null +++ b/server/extension/test/0.5.1/expected/70_ips_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/70_ips_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/85_isodistance_test.out b/server/extension/test/0.5.1/expected/85_isodistance_test.out new file mode 120000 index 0000000..3c81fbf --- /dev/null +++ b/server/extension/test/0.5.1/expected/85_isodistance_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/85_isodistance_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/90_isochrone_test.out b/server/extension/test/0.5.1/expected/90_isochrone_test.out new file mode 120000 index 0000000..596e8ab --- /dev/null +++ b/server/extension/test/0.5.1/expected/90_isochrone_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/90_isochrone_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/95_route_point_to_point_test.out b/server/extension/test/0.5.1/expected/95_route_point_to_point_test.out new file mode 120000 index 0000000..2f0d32c --- /dev/null +++ b/server/extension/test/0.5.1/expected/95_route_point_to_point_test.out @@ -0,0 +1 @@ +../../0.5.0/expected/95_route_point_to_point_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/expected/999_remove_geocoder_api_user_test.out b/server/extension/test/0.5.1/expected/999_remove_geocoder_api_user_test.out new file mode 120000 index 0000000..54a988d --- /dev/null +++ b/server/extension/test/0.5.1/expected/999_remove_geocoder_api_user_test.out @@ -0,0 +1 @@ +../../0.5.0/expected/999_remove_geocoder_api_user_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/00_install_test.sql b/server/extension/test/0.5.1/sql/00_install_test.sql new file mode 100644 index 0000000..85c4fdb --- /dev/null +++ b/server/extension/test/0.5.1/sql/00_install_test.sql @@ -0,0 +1,26 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_dataservices_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.5.1/sql/20_street_test.sql b/server/extension/test/0.5.1/sql/20_street_test.sql new file mode 120000 index 0000000..ca39762 --- /dev/null +++ b/server/extension/test/0.5.1/sql/20_street_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/20_street_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/30_admin0_test.sql b/server/extension/test/0.5.1/sql/30_admin0_test.sql new file mode 120000 index 0000000..f25c0a1 --- /dev/null +++ b/server/extension/test/0.5.1/sql/30_admin0_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/30_admin0_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/40_admin1_test.sql b/server/extension/test/0.5.1/sql/40_admin1_test.sql new file mode 120000 index 0000000..ada2acf --- /dev/null +++ b/server/extension/test/0.5.1/sql/40_admin1_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/40_admin1_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/50_namedplaces_test.sql b/server/extension/test/0.5.1/sql/50_namedplaces_test.sql new file mode 120000 index 0000000..deb2105 --- /dev/null +++ b/server/extension/test/0.5.1/sql/50_namedplaces_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/50_namedplaces_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/60_postalcodes_test.sql b/server/extension/test/0.5.1/sql/60_postalcodes_test.sql new file mode 120000 index 0000000..19eff09 --- /dev/null +++ b/server/extension/test/0.5.1/sql/60_postalcodes_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/60_postalcodes_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/70_ips_test.sql b/server/extension/test/0.5.1/sql/70_ips_test.sql new file mode 120000 index 0000000..7af108a --- /dev/null +++ b/server/extension/test/0.5.1/sql/70_ips_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/70_ips_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/85_isodistance_test.sql b/server/extension/test/0.5.1/sql/85_isodistance_test.sql new file mode 120000 index 0000000..d819b66 --- /dev/null +++ b/server/extension/test/0.5.1/sql/85_isodistance_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/85_isodistance_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/90_isochrone_test.sql b/server/extension/test/0.5.1/sql/90_isochrone_test.sql new file mode 120000 index 0000000..33feb9b --- /dev/null +++ b/server/extension/test/0.5.1/sql/90_isochrone_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/90_isochrone_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/95_route_point_to_point_test.sql b/server/extension/test/0.5.1/sql/95_route_point_to_point_test.sql new file mode 120000 index 0000000..13f56b7 --- /dev/null +++ b/server/extension/test/0.5.1/sql/95_route_point_to_point_test.sql @@ -0,0 +1 @@ +../../0.5.0/sql/95_route_point_to_point_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.1/sql/999_remove_geocoder_api_user_test.sql b/server/extension/test/0.5.1/sql/999_remove_geocoder_api_user_test.sql new file mode 120000 index 0000000..b9b169d --- /dev/null +++ b/server/extension/test/0.5.1/sql/999_remove_geocoder_api_user_test.sql @@ -0,0 +1 @@ +../../0.5.0/sql/999_remove_geocoder_api_user_test.sql \ No newline at end of file From 7237e01601fd51daed236a0ef0ca688837860c72 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 Mar 2016 15:40:37 +0100 Subject: [PATCH 02/10] Change metrics increment naming to reflect service instead of geocoder --- .../sql/0.5.1/100_routing_helper.sql | 8 ++--- .../extension/sql/0.5.1/20_geocode_street.sql | 16 +++++----- server/extension/sql/0.5.1/30_admin0.sql | 8 ++--- server/extension/sql/0.5.1/40_admin1.sql | 16 +++++----- server/extension/sql/0.5.1/50_namedplaces.sql | 24 +++++++------- server/extension/sql/0.5.1/60_postalcodes.sql | 32 +++++++++---------- server/extension/sql/0.5.1/70_ips.sql | 8 ++--- .../sql/0.5.1/80_isolines_helper.sql | 8 ++--- .../cartodb_services/metrics/quota.py | 14 +++----- server/lib/python/cartodb_services/setup.py | 2 +- .../test/test_quota_service.py | 8 ++--- 11 files changed, 69 insertions(+), 75 deletions(-) diff --git a/server/extension/sql/0.5.1/100_routing_helper.sql b/server/extension/sql/0.5.1/100_routing_helper.sql index e23a1c5..26a8880 100644 --- a/server/extension/sql/0.5.1/100_routing_helper.sql +++ b/server/extension/sql/0.5.1/100_routing_helper.sql @@ -39,17 +39,17 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if resp: shape_linestring = polyline_to_linestring(resp.shape) - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return [shape_linestring, resp.length, resp.duration] else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/20_geocode_street.sql b/server/extension/sql/0.5.1/20_geocode_street.sql index de77f62..3c73087 100644 --- a/server/extension/sql/0.5.1/20_geocode_street.sql +++ b/server/extension/sql/0.5.1/20_geocode_street.sql @@ -34,22 +34,22 @@ RETURNS Geometry AS $$ geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -65,20 +65,20 @@ RETURNS Geometry AS $$ geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.1/30_admin0.sql b/server/extension/sql/0.5.1/30_admin0.sql index ec8fe5c..395253b 100644 --- a/server/extension/sql/0.5.1/30_admin0.sql +++ b/server/extension/sql/0.5.1/30_admin0.sql @@ -13,20 +13,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [country_name], 1) result = rv[0]["mypolygon"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.1/40_admin1.sql b/server/extension/sql/0.5.1/40_admin1.sql index dc8e037..011514b 100644 --- a/server/extension/sql/0.5.1/40_admin1.sql +++ b/server/extension/sql/0.5.1/40_admin1.sql @@ -14,20 +14,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [admin1_name], 1) result = rv[0]["mypolygon"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) @@ -46,20 +46,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [admin1_name, country_name], 1) result = rv[0]["mypolygon"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/0.5.1/50_namedplaces.sql b/server/extension/sql/0.5.1/50_namedplaces.sql index 80306cb..0ee069f 100644 --- a/server/extension/sql/0.5.1/50_namedplaces.sql +++ b/server/extension/sql/0.5.1/50_namedplaces.sql @@ -14,20 +14,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [city_name], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; ---- cdb_geocode_namedplace_point(city_name text, country_name text) @@ -46,20 +46,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [city_name, country_name], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) @@ -78,20 +78,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/0.5.1/60_postalcodes.sql b/server/extension/sql/0.5.1/60_postalcodes.sql index 4ba321b..35540a3 100644 --- a/server/extension/sql/0.5.1/60_postalcodes.sql +++ b/server/extension/sql/0.5.1/60_postalcodes.sql @@ -13,20 +13,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [code], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) @@ -44,20 +44,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [code, country], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) @@ -75,20 +75,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [code], 1) result = rv[0]["mypolygon"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) @@ -106,20 +106,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [code, country], 1) result = rv[0]["mypolygon"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/0.5.1/70_ips.sql b/server/extension/sql/0.5.1/70_ips.sql index f76190f..5b70606 100644 --- a/server/extension/sql/0.5.1/70_ips.sql +++ b/server/extension/sql/0.5.1/70_ips.sql @@ -13,20 +13,20 @@ RETURNS Geometry AS $$ rv = plpy.execute(plan, [ip], 1) result = rv[0]["mypoint"] if result: - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() return None except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/0.5.1/80_isolines_helper.sql b/server/extension/sql/0.5.1/80_isolines_helper.sql index eb15963..7eb7cdc 100644 --- a/server/extension/sql/0.5.1/80_isolines_helper.sql +++ b/server/extension/sql/0.5.1/80_isolines_helper.sql @@ -37,18 +37,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ polyline = isoline['geom'] multipolygon = geo_polyline_to_multipolygon(polyline) result.append([source, data_range_n, multipolygon]) - quota_service.increment_success_geocoder_use() + quota_service.increment_success_service_use() quota_service.increment_isolines_service_use(len(resp)) return result else: - quota_service.increment_empty_geocoder_use() + quota_service.increment_empty_service_use() except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) plpy.notice(traceback.format_tb(traceback_)) plpy.error(error_msg) finally: - quota_service.increment_total_geocoder_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index aba6135..33550cc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -18,28 +18,22 @@ class QuotaService: def check_user_quota(self): return self._quota_checker.check() - # TODO - # We are going to change this class to be the generic one and - # create specific for routing and geocoding services but because - # this implies change all the extension functions, we are going to - # make the change in a minor release - - def increment_success_geocoder_use(self, amount=1): + def increment_success_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "success_responses", amount=amount) - def increment_empty_geocoder_use(self, amount=1): + def increment_empty_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "empty_responses", amount=amount) - def increment_failed_geocoder_use(self, amount=1): + def increment_failed_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "fail_responses", amount=amount) - def increment_total_geocoder_use(self, amount=1): + def increment_total_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "total_requests", amount=amount) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 550bfbd..499e833 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.3.1', + version='0.3.2', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index ef63c5a..fbbb4ff 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -63,18 +63,18 @@ class TestQuotaService(TestCase): def test_should_check_user_increment_and_quota_check_correctly(self): qs = self.__build_quota_service('test_user', quota=2) - qs.increment_success_geocoder_use() + qs.increment_success_service_use() assert qs.check_user_quota() is True - qs.increment_success_geocoder_use(amount=2) + qs.increment_success_service_use(amount=2) assert qs.check_user_quota() is False month = date.today().strftime('%Y%m') def test_should_check_org_increment_and_quota_check_correctly(self): qs = self.__build_quota_service('test_user', orgname='test_org', quota=2) - qs.increment_success_geocoder_use() + qs.increment_success_service_use() assert qs.check_user_quota() is True - qs.increment_success_geocoder_use(amount=2) + qs.increment_success_service_use(amount=2) assert qs.check_user_quota() is False month = date.today().strftime('%Y%m') From e86739426144d78a17a6d851dbeca6e7336a2c8c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 Mar 2016 15:46:04 +0100 Subject: [PATCH 03/10] Move the old version files to a folder --- client/Makefile | 9 ++------- .../cdb_dataservices_client--0.0.1--0.1.0.sql | 0 .../cdb_dataservices_client--0.0.1.sql | 0 .../cdb_dataservices_client--0.1.0--0.0.1.sql | 0 .../cdb_dataservices_client--0.1.0--0.2.0.sql | 0 .../cdb_dataservices_client--0.1.0.sql | 0 .../cdb_dataservices_client--0.2.0--0.1.0.sql | 0 .../cdb_dataservices_client--0.2.0.sql | 0 server/extension/.gitignore | 2 ++ server/extension/Makefile | 19 ++++--------------- .../cdb_dataservices_server--0.0.1--0.1.0.sql | 0 .../cdb_dataservices_server--0.0.1.sql | 0 .../cdb_dataservices_server--0.1.0--0.0.1.sql | 0 .../cdb_dataservices_server--0.1.0--0.2.0.sql | 0 .../cdb_dataservices_server--0.1.0.sql | 0 .../cdb_dataservices_server--0.2.0--0.1.0.sql | 0 .../cdb_dataservices_server--0.2.0--0.3.0.sql | 0 .../cdb_dataservices_server--0.2.0.sql | 0 .../cdb_dataservices_server--0.3.0--0.2.0.sql | 0 .../cdb_dataservices_server--0.3.0--0.4.0.sql | 0 .../cdb_dataservices_server--0.3.0.sql | 0 .../cdb_dataservices_server--0.4.0--0.3.0.sql | 0 .../cdb_dataservices_server--0.4.0--0.5.0.sql | 0 .../cdb_dataservices_server--0.4.0.sql | 0 .../cdb_dataservices_server--0.5.0--0.4.0.sql | 0 .../cdb_dataservices_server--0.5.0.sql | 0 26 files changed, 8 insertions(+), 22 deletions(-) rename client/{ => old_versions}/cdb_dataservices_client--0.0.1--0.1.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.0.1.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.1.0--0.0.1.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.1.0--0.2.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.1.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.2.0--0.1.0.sql (100%) rename client/{ => old_versions}/cdb_dataservices_client--0.2.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.0.1--0.1.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.0.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.1.0--0.0.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.1.0--0.2.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.1.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.2.0--0.1.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.2.0--0.3.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.2.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.3.0--0.2.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.3.0--0.4.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.3.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.4.0--0.3.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.4.0--0.5.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.4.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.5.0--0.4.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.5.0.sql (100%) diff --git a/client/Makefile b/client/Makefile index a15da74..074cd73 100644 --- a/client/Makefile +++ b/client/Makefile @@ -5,19 +5,14 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/defau # The new version to be generated from templates NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql +OLD_VERSIONS = $(wildcard old_versions/*.sql) # DATA is a special variable used by postgres build infrastructure # These are the files to be installed in the server shared dir, # for installation from scratch, upgrades and downgrades. # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ - cdb_dataservices_client--0.0.1.sql \ - cdb_dataservices_client--0.1.0.sql \ - cdb_dataservices_client--0.2.0.sql \ - cdb_dataservices_client--0.1.0--0.0.1.sql \ - cdb_dataservices_client--0.0.1--0.1.0.sql \ - cdb_dataservices_client--0.2.0--0.1.0.sql \ - cdb_dataservices_client--0.1.0--0.2.0.sql \ + $(OLD_VERSIONS) \ cdb_dataservices_client--0.2.0--0.3.0.sql \ cdb_dataservices_client--0.3.0--0.2.0.sql diff --git a/client/cdb_dataservices_client--0.0.1--0.1.0.sql b/client/old_versions/cdb_dataservices_client--0.0.1--0.1.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.0.1--0.1.0.sql rename to client/old_versions/cdb_dataservices_client--0.0.1--0.1.0.sql diff --git a/client/cdb_dataservices_client--0.0.1.sql b/client/old_versions/cdb_dataservices_client--0.0.1.sql similarity index 100% rename from client/cdb_dataservices_client--0.0.1.sql rename to client/old_versions/cdb_dataservices_client--0.0.1.sql diff --git a/client/cdb_dataservices_client--0.1.0--0.0.1.sql b/client/old_versions/cdb_dataservices_client--0.1.0--0.0.1.sql similarity index 100% rename from client/cdb_dataservices_client--0.1.0--0.0.1.sql rename to client/old_versions/cdb_dataservices_client--0.1.0--0.0.1.sql diff --git a/client/cdb_dataservices_client--0.1.0--0.2.0.sql b/client/old_versions/cdb_dataservices_client--0.1.0--0.2.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.1.0--0.2.0.sql rename to client/old_versions/cdb_dataservices_client--0.1.0--0.2.0.sql diff --git a/client/cdb_dataservices_client--0.1.0.sql b/client/old_versions/cdb_dataservices_client--0.1.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.1.0.sql rename to client/old_versions/cdb_dataservices_client--0.1.0.sql diff --git a/client/cdb_dataservices_client--0.2.0--0.1.0.sql b/client/old_versions/cdb_dataservices_client--0.2.0--0.1.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.2.0--0.1.0.sql rename to client/old_versions/cdb_dataservices_client--0.2.0--0.1.0.sql diff --git a/client/cdb_dataservices_client--0.2.0.sql b/client/old_versions/cdb_dataservices_client--0.2.0.sql similarity index 100% rename from client/cdb_dataservices_client--0.2.0.sql rename to client/old_versions/cdb_dataservices_client--0.2.0.sql diff --git a/server/extension/.gitignore b/server/extension/.gitignore index 3990497..f103abf 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -5,3 +5,5 @@ cdb_geocoder_server--0.0.1.sql cdb_geocoder_server--0.1.0.sql cdb_geocoder_server--0.2.0.sql cdb_geocoder_server--0.3.0.sql +cdb_geocoder_server--0.4.0.sql +cdb_geocoder_server--0.5.0.sql \ No newline at end of file diff --git a/server/extension/Makefile b/server/extension/Makefile index 0d8091c..40e6163 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -5,27 +5,16 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/defau # The new version to be generated from templates NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql +OLD_VERSIONS = $(wildcard old_versions/*.sql) # DATA is a special variable used by postgres build infrastructure # These are the files to be installed in the server shared dir, # for installation from scratch, upgrades and downgrades. # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ - cdb_dataservices_server--0.0.1.sql \ - cdb_dataservices_server--0.1.0.sql \ - cdb_dataservices_server--0.2.0.sql \ - cdb_dataservices_server--0.3.0.sql \ - cdb_dataservices_server--0.4.0.sql \ - cdb_dataservices_server--0.1.0--0.0.1.sql \ - cdb_dataservices_server--0.0.1--0.1.0.sql \ - cdb_dataservices_server--0.2.0--0.1.0.sql \ - cdb_dataservices_server--0.1.0--0.2.0.sql \ - cdb_dataservices_server--0.2.0--0.3.0.sql \ - cdb_dataservices_server--0.3.0--0.2.0.sql \ - cdb_dataservices_server--0.3.0--0.4.0.sql \ - cdb_dataservices_server--0.4.0--0.3.0.sql \ - cdb_dataservices_server--0.5.0--0.4.0.sql \ - cdb_dataservices_server--0.4.0--0.5.0.sql + $(OLD_VERSIONS) \ + cdb_dataservices_server--0.5.1--0.5.0.sql \ + cdb_dataservices_server--0.5.0--0.5.1.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) TEST_DIR = test/$(EXTVERSION) diff --git a/server/extension/cdb_dataservices_server--0.0.1--0.1.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.0.1--0.1.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.0.1--0.1.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.0.1--0.1.0.sql diff --git a/server/extension/cdb_dataservices_server--0.0.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.0.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.0.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.0.1.sql diff --git a/server/extension/cdb_dataservices_server--0.1.0--0.0.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.1.0--0.0.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.1.0--0.0.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.1.0--0.0.1.sql diff --git a/server/extension/cdb_dataservices_server--0.1.0--0.2.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.1.0--0.2.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.1.0--0.2.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.1.0--0.2.0.sql diff --git a/server/extension/cdb_dataservices_server--0.1.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.1.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.1.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.1.0.sql diff --git a/server/extension/cdb_dataservices_server--0.2.0--0.1.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.2.0--0.1.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.2.0--0.1.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.2.0--0.1.0.sql diff --git a/server/extension/cdb_dataservices_server--0.2.0--0.3.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.2.0--0.3.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.2.0--0.3.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.2.0--0.3.0.sql diff --git a/server/extension/cdb_dataservices_server--0.2.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.2.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.2.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.2.0.sql diff --git a/server/extension/cdb_dataservices_server--0.3.0--0.2.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.3.0--0.2.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.3.0--0.2.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.3.0--0.2.0.sql diff --git a/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.3.0--0.4.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.3.0--0.4.0.sql diff --git a/server/extension/cdb_dataservices_server--0.3.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.3.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.3.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.3.0.sql diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.3.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.4.0--0.3.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.4.0--0.3.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.4.0--0.3.0.sql diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.4.0--0.5.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.4.0--0.5.0.sql diff --git a/server/extension/cdb_dataservices_server--0.4.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.4.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.4.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.4.0.sql diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.5.0--0.4.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.5.0--0.4.0.sql diff --git a/server/extension/cdb_dataservices_server--0.5.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.5.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.5.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.5.0.sql From 0b4d1bb17f6989405a4b8896c45ccdedb9635926 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 Mar 2016 15:46:27 +0100 Subject: [PATCH 04/10] Upgrade files for the 0.5.1 version of the server --- .../cdb_dataservices_server--0.5.0--0.5.1.sql | 508 ++++++++ .../cdb_dataservices_server--0.5.1--0.5.0.sql | 508 ++++++++ .../cdb_dataservices_server--0.5.1.sql | 1019 +++++++++++++++++ 3 files changed, 2035 insertions(+) create mode 100644 server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.5.1.sql diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql new file mode 100644 index 0000000..d39dbdb --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql @@ -0,0 +1,508 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql new file mode 100644 index 0000000..dab17ee --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql @@ -0,0 +1,508 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_geocoder_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.1.sql new file mode 100644 index 0000000..75b7380 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.5.1.sql @@ -0,0 +1,1019 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_point_to_point($1, $2, $3, $4, $5, $6, $7) as route;", ["text", "text", "geometry(Point, 4326)", "geometry(Point, 4326)", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_master_id text, + redis_host text, + redis_port int, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + redis_conf_params = { + "redis_host": params['redis_host'], + "redis_port": params['redis_port'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + metadata_config_params['redis_port'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + metrics_config_params['redis_port'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; From ac1627b5c513f0323e3059a66d7f12251d2ed538 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 7 Mar 2016 17:54:10 +0100 Subject: [PATCH 05/10] Logger for geocoder metrics --- .../cartodb_services/metrics/config.py | 33 ++++++++- .../cartodb_services/metrics/log.py | 74 +++++++++++++++++++ .../cartodb_services/metrics/quota.py | 19 ++++- 3 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/metrics/log.py diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d2b3b80..0f318a0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -146,6 +146,18 @@ class InternalGeocoderConfig(ServiceConfig): def service_type(self): return 'geocoder_internal' + @property + def is_high_resolution(self): + return False + + @property + def cost_per_hit(self): + return 0 + + @property + def geocoding_quota(self): + return None + class GeocoderConfig(ServiceConfig): @@ -168,6 +180,7 @@ class GeocoderConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' + LOG_PATH = '/var/log/postgresql/geocodings.log' def __init__(self, redis_connection, username, orgname=None, heremaps_app_id=None, heremaps_app_code=None): @@ -256,7 +269,10 @@ class GeocoderConfig(ServiceConfig): @property def geocoding_quota(self): - return self._geocoding_quota + if self.heremaps_geocoder: + return self._geocoding_quota + else: + return None @property def soft_geocoding_limit(self): @@ -273,3 +289,18 @@ class GeocoderConfig(ServiceConfig): @property def heremaps_app_code(self): return self._heremaps_app_code + + @property + def is_high_resolution(self): + return True + + @property + def cost_per_hit(self): + if self.heremaps_geocoder: + return 1 + else: + return 0 + + @property + def log_path(self): + return self.LOG_PATH diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py new file mode 100644 index 0000000..82dacb2 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -0,0 +1,74 @@ + +import abc +import json +import re + + +class LoggerFactory: + + @classmethod + def build(self, service_config): + if re.match('geocoder_*', service_config.service_type): + return GeocoderLogger(service_config) + else: + return None + + +class Logger(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, file_path): + self._file_path = file_path + + def dump_to_file(self, data): + with open(self._file_path, 'a') as logfile: + json.dump(data, logfile) + + @abc.abstractproperty + def log(self, **data): + raise NotImplementedError('log method must be defined') + + +class GeocoderLogger(Logger): + + def __init__(self, service_config): + super(GeocoderLogger, self).__init__(service_config.log_path) + self._service_config = service_config + + def log(self, **data): + dump_data = self._dump_data(**data) + self.dump_to_file(dump_data) + + def _dump_data(self, **data): + if data['success']: + cost = self._service_config.cost_per_hit + failed_rows = 0 + successful_rows = 1 + else: + cost = 0 + failed_rows = 1 + successful_rows = 0 + + if self._service_config.is_high_resolution: + kind = 'high-resolution' + else: + kind = 'internal' + + return { + "batched": False, + "cache_hits": 0, # Always 0 because no cache involved + # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211 + "cost": cost, + "created_at": datetime.now().isoformat(), + "failed_rows": failed_rows, + "geocoder_type": self._service_config.service_type, + "kind": kind, + "processable_rows": 1, + "processed_rows": successful_rows, + "real_rows": successful_rows, + "success": data['success'], + "successful_rows": successful_rows, + "used_credits": 0, + "username": self._service_config.username, + "organization": self._service_config.organization + } diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 33550cc..7eefd11 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,19 +1,20 @@ from user import UserMetricsService +from log import LoggerFactory from datetime import date import re class QuotaService: """ Class to manage all the quota operation for - the Geocoder SQL API Extension """ + the Dataservices SQL API Extension """ def __init__(self, user_service_config, redis_connection): self._user_service_config = user_service_config - # TODO First step to extract to a factory if needed in the future self._quota_checker = QuotaChecker(user_service_config, redis_connection) - self._user_service = UserMetricsService( - self._user_service_config, redis_connection) + self._user_service = UserMetricsService(self._user_service_config, + redis_connection) + self._logger = LoggerFactory.build(user_service_config) def check_user_quota(self): return self._quota_checker.check() @@ -22,16 +23,19 @@ class QuotaService: self._user_service.increment_service_use( self._user_service_config.service_type, "success_responses", amount=amount) + self._log_service_process("success") def increment_empty_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "empty_responses", amount=amount) + self._log_service_process("empty") def increment_failed_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "fail_responses", amount=amount) + self._log_service_process("fail") def increment_total_service_use(self, amount=1): self._user_service.increment_service_use( @@ -43,6 +47,13 @@ class QuotaService: self._user_service_config.service_type, "isolines_generated", amount=amount) + def _log_service_process(self, event): + if self._logger: + if event is 'success' or event is 'empty': + self._logger.log(success=True) + elif event is 'empty': + self._logger.log(success=False) + class QuotaChecker: From b97e83841667e9fed1f2dde63574d941f0ef681a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 8 Mar 2016 12:24:27 +0100 Subject: [PATCH 06/10] Refactor to move logic from SQL functions to Python - Moved the logic the retrieve the redis connection params to RedisDBConfig class - Moved the logic that retrieve the services configuration to ServicesDBConfig --- .../cdb_dataservices_server--0.5.0--0.5.1.sql | 65 +++++++- .../cdb_dataservices_server--0.5.1--0.5.0.sql | 140 +++++++++++++++++- .../cdb_dataservices_server--0.5.1.sql | 91 +----------- .../extension/sql/0.5.1/10_redis_helper.sql | 19 ++- .../extension/sql/0.5.1/15_config_helper.sql | 45 +++++- .../cartodb_services/metrics/config.py | 88 ++++++++--- .../cartodb_services/metrics/log.py | 2 +- .../cartodb_services/tools/__init__.py | 2 +- .../cartodb_services/tools/redis_tools.py | 87 +++++++++-- .../cartodb_services/test/test_config.py | 18 +-- .../cartodb_services/test/test_helper.py | 16 ++ .../test/test_quota_service.py | 6 +- .../test/test_user_service.py | 7 +- 13 files changed, 445 insertions(+), 141 deletions(-) mode change 120000 => 100644 server/extension/sql/0.5.1/10_redis_helper.sql mode change 120000 => 100644 server/extension/sql/0.5.1/15_config_helper.sql diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql index d39dbdb..647eae3 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql @@ -1,3 +1,66 @@ +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_redis_conf_v2(text); +DROP TYPE IF EXISTS cdb_dataservices_server._redis_conf_params; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -505,4 +568,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ plpy.error(error_msg) finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql index dab17ee..a31e055 100644 --- a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql @@ -1,3 +1,141 @@ +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_master_id text, + redis_host text, + redis_port int, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + redis_conf_params = { + "redis_host": params['redis_host'], + "redis_port": params['redis_port'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + metadata_config_params['redis_port'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + metrics_config_params['redis_port'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -505,4 +643,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ plpy.error(error_msg) finally: quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.1.sql index 75b7380..ad46cf6 100644 --- a/server/extension/cdb_dataservices_server--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.1.sql @@ -74,38 +74,6 @@ RETURNS cdb_dataservices_server.simple_route AS $$ result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; -CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( - sentinel_master_id text, - redis_host text, - redis_port int, - redis_db text, - timeout float -); - --- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) -RETURNS cdb_dataservices_server._redis_conf_params AS $$ - conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) - conf = plpy.execute(conf_query)[0]['conf'] - if conf is None: - plpy.error("There is no redis configuration defined") - else: - import json - params = json.loads(conf) - redis_conf_params = { - "redis_host": params['redis_host'], - "redis_port": params['redis_port'], - "timeout": params['timeout'], - "redis_db": params['redis_db'] - } - if "sentinel_master_id" in params: - redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] - else: - redis_conf_params["sentinel_master_id"] = None - - return redis_conf_params -$$ LANGUAGE plpythonu; - -- Get the connection to redis from cache or create a new one CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) RETURNS boolean AS $$ @@ -113,23 +81,11 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - from cartodb_services.tools import RedisConnection - metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, - c.redis_port, c.timeout, c.redis_db - from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, - c.redis_port, c.timeout, c.redis_db - from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], - metadata_config_params['redis_host'], - metadata_config_params['redis_port'], - timeout=metadata_config_params['timeout'], - redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], - metrics_config_params['redis_host'], - metrics_config_params['redis_port'], - timeout=metrics_config_params['timeout'], - redis_db=metrics_config_params['redis_db']).redis_connection() + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() GD[cache_key] = { 'redis_metadata_connection': redis_metadata_connection, 'redis_metrics_connection': redis_metrics_connection, @@ -143,21 +99,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import GeocoderConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] - if not heremaps_conf_json: - heremaps_app_id = None - heremaps_app_code = None - else: - heremaps_conf = json.loads(heremaps_conf_json) - heremaps_app_id = heremaps_conf['app_id'] - heremaps_app_code = heremaps_conf['app_code'] - geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) GD[cache_key] = geocoder_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -169,21 +114,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import IsolinesRoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] - if not heremaps_conf_json: - heremaps_app_id = None - heremaps_app_code = None - else: - heremaps_conf = json.loads(heremaps_conf_json) - heremaps_app_id = heremaps_conf['app_id'] - heremaps_app_code = heremaps_conf['app_code'] - isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = isolines_routing_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -195,19 +129,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import RoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] - if not mapzen_conf_json: - mapzen_app_key = None - else: - mapzen_conf = json.loads(mapzen_conf_json) - mapzen_app_key = mapzen_conf['routing_app_key'] - routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = routing_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/10_redis_helper.sql b/server/extension/sql/0.5.1/10_redis_helper.sql deleted file mode 120000 index 71e8ff6..0000000 --- a/server/extension/sql/0.5.1/10_redis_helper.sql +++ /dev/null @@ -1 +0,0 @@ -../0.5.0/10_redis_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/10_redis_helper.sql b/server/extension/sql/0.5.1/10_redis_helper.sql new file mode 100644 index 0000000..e010088 --- /dev/null +++ b/server/extension/sql/0.5.1/10_redis_helper.sql @@ -0,0 +1,18 @@ +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/15_config_helper.sql b/server/extension/sql/0.5.1/15_config_helper.sql deleted file mode 120000 index fdce38c..0000000 --- a/server/extension/sql/0.5.1/15_config_helper.sql +++ /dev/null @@ -1 +0,0 @@ -../0.5.0/15_config_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/15_config_helper.sql b/server/extension/sql/0.5.1/15_config_helper.sql new file mode 100644 index 0000000..d39a4fc --- /dev/null +++ b/server/extension/sql/0.5.1/15_config_helper.sql @@ -0,0 +1,44 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 0f318a0..129e27e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -27,6 +27,7 @@ class ServiceConfig(object): def organization(self): return self._orgname + class RoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['username', 'orgname', 'mapzen_app_key'] @@ -34,11 +35,11 @@ class RoutingConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' - def __init__(self, redis_connection, username, orgname=None, - mapzen_app_key=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, username, - orgname) - self._mapzen_app_key = mapzen_app_key + orgname) + db_config = ServicesDBConfig(db_conn) + self._mapzen_app_key = db_config.mapzen_routing_app_key @property def service_type(self): @@ -53,7 +54,8 @@ class IsolinesRoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_app_id', 'heremaps_app_code', 'geocoder_type'] + 'heremaps_app_id', 'heremaps_app_code', + 'geocoder_type'] NOKIA_APP_ID_KEY = 'heremaps_app_id' NOKIA_APP_CODE_KEY = 'heremaps_app_code' QUOTA_KEY = 'here_isolines_quota' @@ -64,24 +66,22 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_TYPE_KEY = 'geocoder_type' GOOGLE_GEOCODER = 'google' - def __init__(self, redis_connection, username, orgname=None, - heremaps_app_id=None, heremaps_app_code=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, username, orgname) - config = self.__get_user_config(username, orgname, heremaps_app_id, - heremaps_app_code) + db_config = ServicesDBConfig(db_conn) + config = self.__get_user_config(username, orgname, db_config) filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname=None, heremaps_app_id=None, - heremaps_app_code=None): + def __get_user_config(self, username, orgname, db_config): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_APP_ID_KEY] = heremaps_app_id - user_config[self.NOKIA_APP_CODE_KEY] = heremaps_app_code + user_config[self.NOKIA_APP_ID_KEY] = db_config.heremaps_app_id + user_config[self.NOKIA_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -182,25 +182,23 @@ class GeocoderConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' LOG_PATH = '/var/log/postgresql/geocodings.log' - def __init__(self, redis_connection, username, orgname=None, - heremaps_app_id=None, heremaps_app_code=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(GeocoderConfig, self).__init__(redis_connection, username, orgname) - config = self.__get_user_config(username, orgname, heremaps_app_id, - heremaps_app_code) + db_config = ServicesDBConfig(db_conn) + config = self.__get_user_config(username, orgname, db_config) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} self.__check_config(filtered_config) self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname=None, heremaps_app_id=None, - heremaps_app_code=None): + def __get_user_config(self, username, orgname, db_config): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id - user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code + user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = db_config.heremaps_app_id + user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -304,3 +302,51 @@ class GeocoderConfig(ServiceConfig): @property def log_path(self): return self.LOG_PATH + + +class ServicesDBConfig: + + def __init__(self, db_conn): + self._db_conn = db_conn + return self._build() + + def _build(self): + self._get_here_config() + self._get_mapzen_config() + + def _get_here_config(self): + heremaps_conf_json = self._get_conf('heremaps_conf') + if not heremaps_conf_json: + raise ConfigException('Here maps configuration missing') + else: + heremaps_conf = json.loads(heremaps_conf_json) + self._heremaps_app_id = heremaps_conf['app_id'] + self._heremaps_app_code = heremaps_conf['app_code'] + + def _get_mapzen_config(self): + mapzen_conf_json = self._get_conf('mapzen_conf') + if not mapzen_conf_json: + raise ConfigException('Mapzen configuration missing') + else: + mapzen_conf = json.loads(mapzen_conf_json) + self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] + + def _get_conf(self, key): + try: + sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) + conf = self._db_conn.execute(sql, 1) + return conf[0]['conf'] + except: + raise ConfigException("Malformed config for {0}".format(key)) + + @property + def heremaps_app_id(self): + return self._heremaps_app_id + + @property + def heremaps_app_code(self): + return self._heremaps_app_code + + @property + def mapzen_routing_app_key(self): + return self._mapzen_routing_app_key diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 82dacb2..b4b8d2b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -1,4 +1,4 @@ - +from datetime import datetime import abc import json import re diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index dbf91b5..d884632 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,3 +1,3 @@ -from redis_tools import RedisConnection +from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index cc04f9b..637e81b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -1,32 +1,87 @@ from redis.sentinel import Sentinel from redis import StrictRedis +import json class RedisConnection: - REDIS_DEFAULT_USER_DB = 5 - REDIS_DEFAULT_TIMEOUT = 2 #seconds - - def __init__(self, sentinel_master_id, redis_host, redis_port, - redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.redis_host = redis_host - self.redis_port = redis_port - self.sentinel_master_id = sentinel_master_id - self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT - self.redis_db = redis_db + def __init__(self, config): + self._config = config def redis_connection(self): return self.__create_redis_connection() def __create_redis_connection(self): - if self.sentinel_master_id == None: - return StrictRedis(host=self.redis_host, port=self.redis_port, db=self.redis_db) - else: - sentinel = Sentinel([(self.redis_host, - self.redis_port)], - socket_timeout=self.timeout) + if self._config.sentinel_id: + sentinel = Sentinel([(self._config.host, + self._config.port)], + socket_timeout=self._config.timeout) return sentinel.master_for( self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db ) + else: + conn = StrictRedis(host=self._config.host, port=self._config.port, + db=self._config.db) + return conn + + +class RedisDBConfig: + + DEFAULT_USER_DB = 5 + DEFAULT_TIMEOUT = 2 # seconds + + def __init__(self, key, db_conn): + self._db_conn = db_conn + return self._build(key) + + def _build(self, key): + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format( + key) + conf = self._db_conn.execute(conf_query)[0]['conf'] + if conf is None: + raise "There is no redis configuration defined" + else: + params = json.loads(conf) + self._host = params['redis_host'] + self._port = params['redis_port'] + + if "timeout" in params: + self._timeout = params['timeout'] + else: + self._timeout = self.DEFAULT_TIMEOUT + + if "redis_db" in params: + self._db = params['redis_db'] + else: + self._db = self.DEFAULT_USER_DB + + if "sentinel_master_id" in params: + self._sentinel_id = params["sentinel_master_id"] + else: + self._sentinel_id = None + + def __str__(self): + return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format( + self.host, self.port, self.sentinel_id, self.db) + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def timeout(self): + return self._timeout + + @property + def db(self): + return self._db + + @property + def sentinel_id(self): + return self._sentinel_id diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 7588427..b92dc31 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -10,12 +10,12 @@ class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() + self.plpy_mock = test_helper.build_plpy_mock() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, - 'test_user', None, - 'nokia_id', 'nokia_cod') + geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + 'test_user', None) assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 assert geocoder_config.soft_geocoding_limit is False @@ -25,17 +25,17 @@ class TestConfig(TestCase): test_helper.build_redis_user_config(self.redis_conn, 'test_user') test_helper.build_redis_org_config(self.redis_conn, 'test_org', quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, - 'test_user', 'test_org', - 'nokia_id', 'nokia_cod') + geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + 'test_user', 'test_org') assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 assert geocoder_config.soft_geocoding_limit is False assert geocoder_config.period_end_date.date() == yesterday.date() - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): + def test_should_raise_exception_when_missing_parameters(self): + plpy_mock = test_helper.build_plpy_mock(empty=True) test_helper.build_redis_user_config(self.redis_conn, 'test_user') assert_raises(ConfigException, GeocoderConfig, - self.redis_conn, 'test_user', - None, None, None) + self.redis_conn, plpy_mock, 'test_user', + None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 9f1f50f..e732077 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,4 +1,5 @@ from datetime import datetime, date +from mock import Mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -31,3 +32,18 @@ def increment_geocoder_uses(redis_conn, username, orgname=None, redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, service, metric, yearmonth) redis_conn.zincrby(redis_name, date.day, amount) + + +def build_plpy_mock(empty=False): + plpy_mock = Mock() + if not empty: + plpy_mock.execute.side_effect = _plpy_execute_side_effect + + return plpy_mock + + +def _plpy_execute_side_effect(*args, **kwargs): + if args[0] == 'SELECT cartodb.CDB_Conf_GetConf(heremaps_conf) as conf': + return [{'conf': '{"app_id": "app_id", "app_code": "code"}'}] + elif args[0] == 'SELECT cartodb.CDB_Conf_GetConf(mapzen_conf) as conf': + return [{'conf': '{"routing_app_key": "app_key"}'}] diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index fbbb4ff..9df7f88 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -88,9 +88,9 @@ class TestQuotaService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') + plpy_mock = test_helper.build_plpy_mock() + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + username, orgname) return QuotaService(geocoder_config, redis_connection = self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 8535ec3..9066643 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -4,6 +4,7 @@ from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import GeocoderConfig from datetime import datetime, date from unittest import TestCase +from mock import Mock from nose.tools import assert_raises from datetime import timedelta @@ -84,7 +85,7 @@ class TestUserService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') + plpy_mock = test_helper.build_plpy_mock() + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + username, orgname,) return UserMetricsService(geocoder_config, self.redis_conn) From c293b55c3ccdd84b3ed1abd782180fc1a9a62a81 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 8 Mar 2016 15:53:15 +0100 Subject: [PATCH 07/10] Added log and cost_per_hit config to the db config --- .../test/0.5.1/sql/00_install_test.sql | 2 + .../cartodb_services/metrics/config.py | 63 +++++++++++-------- .../cartodb_services/metrics/log.py | 1 + server/lib/python/cartodb_services/path | 4 ++ .../cartodb_services/test/test_helper.py | 8 ++- 5 files changed, 49 insertions(+), 29 deletions(-) create mode 100644 server/lib/python/cartodb_services/path diff --git a/server/extension/test/0.5.1/sql/00_install_test.sql b/server/extension/test/0.5.1/sql/00_install_test.sql index 85c4fdb..1a296de 100644 --- a/server/extension/test/0.5.1/sql/00_install_test.sql +++ b/server/extension/test/0.5.1/sql/00_install_test.sql @@ -11,7 +11,9 @@ CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/var/log/postgresql/geocodings.log"}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 129e27e..d0670d7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -70,18 +70,16 @@ class IsolinesRoutingConfig(ServiceConfig): super(IsolinesRoutingConfig, self).__init__(redis_connection, username, orgname) db_config = ServicesDBConfig(db_conn) - config = self.__get_user_config(username, orgname, db_config) + config = self.__get_user_config(username, orgname) filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} - self.__parse_config(filtered_config) + self.__parse_config(filtered_config, db_config) - def __get_user_config(self, username, orgname, db_config): + def __get_user_config(self, username, orgname): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_APP_ID_KEY] = db_config.heremaps_app_id - user_config[self.NOKIA_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -96,7 +94,7 @@ class IsolinesRoutingConfig(ServiceConfig): user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] - def __parse_config(self, filtered_config): + def __parse_config(self, filtered_config, db_config): self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower() self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) @@ -104,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig): self._soft_isolines_limit = True else: self._soft_isolines_limit = False - self._heremaps_app_id = filtered_config[self.NOKIA_APP_ID_KEY] - self._heremaps_app_code = filtered_config[self.NOKIA_APP_CODE_KEY] + self._heremaps_app_id = db_config.heremaps_app_id + self._heremaps_app_code = db_config.heremaps_app_code @property def service_type(self): @@ -166,8 +164,7 @@ class GeocoderConfig(ServiceConfig): 'geocoder_type', 'period_end_date', 'heremaps_app_id', 'heremaps_app_code', 'username', 'orgname'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit', - 'heremaps_app_id', 'heremaps_app_code'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' @@ -180,25 +177,22 @@ class GeocoderConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' - LOG_PATH = '/var/log/postgresql/geocodings.log' def __init__(self, redis_connection, db_conn, username, orgname=None): super(GeocoderConfig, self).__init__(redis_connection, username, orgname) db_config = ServicesDBConfig(db_conn) - config = self.__get_user_config(username, orgname, db_config) + config = self.__get_user_config(username, orgname) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} + self.__parse_config(filtered_config, db_config) self.__check_config(filtered_config) - self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname, db_config): + def __get_user_config(self, username, orgname): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = db_config.heremaps_app_id - user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -219,28 +213,29 @@ class GeocoderConfig(ServiceConfig): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") - if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]: - raise ConfigException("""Nokia geocoder configuration is missing. Check it please""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") return True - def __parse_config(self, filtered_config): + def __parse_config(self, filtered_config, db_config): self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + self._log_path = db_config.geocoder_log_path if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: self._soft_geocoding_limit = False if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] - self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] + self._heremaps_app_id = db_config.heremaps_app_id + self._heremaps_app_code = db_config.heremaps_app_code + self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] + self._cost_per_hit = 0 @property def service_type(self): @@ -294,14 +289,11 @@ class GeocoderConfig(ServiceConfig): @property def cost_per_hit(self): - if self.heremaps_geocoder: - return 1 - else: - return 0 + self._cost_per_hit @property def log_path(self): - return self.LOG_PATH + return self._log_path class ServicesDBConfig: @@ -313,6 +305,7 @@ class ServicesDBConfig: def _build(self): self._get_here_config() self._get_mapzen_config() + self._get_logger_config() def _get_here_config(self): heremaps_conf_json = self._get_conf('heremaps_conf') @@ -322,6 +315,8 @@ class ServicesDBConfig: heremaps_conf = json.loads(heremaps_conf_json) self._heremaps_app_id = heremaps_conf['app_id'] self._heremaps_app_code = heremaps_conf['app_code'] + self._heremaps_geocoder_cost_per_hit = heremaps_conf[ + 'geocoder_cost_per_hit'] def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -331,6 +326,14 @@ class ServicesDBConfig: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] + def _get_logger_config(self): + logger_conf_json = self._get_conf('logger_conf') + if not logger_conf_json: + raise ConfigException('Logger configuration missing') + else: + logger_conf = json.loads(logger_conf_json) + self._geocoder_log_path = logger_conf['geocoder_log_path'] + def _get_conf(self, key): try: sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) @@ -347,6 +350,14 @@ class ServicesDBConfig: def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_geocoder_cost_per_hit(self): + return self._heremaps_geocoder_cost_per_hit + @property def mapzen_routing_app_key(self): return self._mapzen_routing_app_key + + @property + def geocoder_log_path(self): + return self._geocoder_log_path diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index b4b8d2b..29759e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -23,6 +23,7 @@ class Logger(object): def dump_to_file(self, data): with open(self._file_path, 'a') as logfile: json.dump(data, logfile) + logfile.write('\n') @abc.abstractproperty def log(self, **data): diff --git a/server/lib/python/cartodb_services/path b/server/lib/python/cartodb_services/path new file mode 100644 index 0000000..ce15f7f --- /dev/null +++ b/server/lib/python/cartodb_services/path @@ -0,0 +1,4 @@ +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.316618", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.318872", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.322009", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.323941", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index e732077..7aaebd3 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -43,7 +43,9 @@ def build_plpy_mock(empty=False): def _plpy_execute_side_effect(*args, **kwargs): - if args[0] == 'SELECT cartodb.CDB_Conf_GetConf(heremaps_conf) as conf': - return [{'conf': '{"app_id": "app_id", "app_code": "code"}'}] - elif args[0] == 'SELECT cartodb.CDB_Conf_GetConf(mapzen_conf) as conf': + if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": + return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": return [{'conf': '{"routing_app_key": "app_key"}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": + return [{'conf': '{"geocoder_log_path": "path"}'}] From bd2e3802259d587f3309e961fee5fa2365915b1a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 8 Mar 2016 16:08:56 +0100 Subject: [PATCH 08/10] Add logger for the internal geocoder too --- .../cdb_dataservices_server--0.5.0--0.5.1.sql | 22 +++++++++---------- .../cdb_dataservices_server--0.5.1.sql | 22 +++++++++---------- server/extension/sql/0.5.1/30_admin0.sql | 2 +- server/extension/sql/0.5.1/40_admin1.sql | 4 ++-- server/extension/sql/0.5.1/50_namedplaces.sql | 6 ++--- server/extension/sql/0.5.1/60_postalcodes.sql | 8 +++---- server/extension/sql/0.5.1/70_ips.sql | 2 +- .../test/0.5.0/expected/00_install_test.out | 14 +++++++++++- .../test/0.5.1/sql/00_install_test.sql | 4 ++-- .../cartodb_services/metrics/config.py | 12 +++++++--- server/lib/python/cartodb_services/path | 4 ---- .../cartodb_services/test/test_helper.py | 2 +- 12 files changed, 58 insertions(+), 44 deletions(-) delete mode 100644 server/lib/python/cartodb_services/path diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql index 647eae3..ca41e78 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql @@ -134,7 +134,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -165,7 +165,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -196,7 +196,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -227,7 +227,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -258,7 +258,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -289,7 +289,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -320,7 +320,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -351,7 +351,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -382,7 +382,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -413,7 +413,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -444,7 +444,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/cdb_dataservices_server--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.1.sql index ad46cf6..e20e4a2 100644 --- a/server/extension/cdb_dataservices_server--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.1.sql @@ -227,7 +227,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -278,7 +278,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -310,7 +310,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -395,7 +395,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -427,7 +427,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -459,7 +459,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -558,7 +558,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -589,7 +589,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -620,7 +620,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -651,7 +651,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -777,7 +777,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/sql/0.5.1/30_admin0.sql b/server/extension/sql/0.5.1/30_admin0.sql index 395253b..a626e2a 100644 --- a/server/extension/sql/0.5.1/30_admin0.sql +++ b/server/extension/sql/0.5.1/30_admin0.sql @@ -5,7 +5,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/sql/0.5.1/40_admin1.sql b/server/extension/sql/0.5.1/40_admin1.sql index 011514b..457fd6e 100644 --- a/server/extension/sql/0.5.1/40_admin1.sql +++ b/server/extension/sql/0.5.1/40_admin1.sql @@ -6,7 +6,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -38,7 +38,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/sql/0.5.1/50_namedplaces.sql b/server/extension/sql/0.5.1/50_namedplaces.sql index 0ee069f..41fbd05 100644 --- a/server/extension/sql/0.5.1/50_namedplaces.sql +++ b/server/extension/sql/0.5.1/50_namedplaces.sql @@ -6,7 +6,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -38,7 +38,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -70,7 +70,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/sql/0.5.1/60_postalcodes.sql b/server/extension/sql/0.5.1/60_postalcodes.sql index 35540a3..dfdc004 100644 --- a/server/extension/sql/0.5.1/60_postalcodes.sql +++ b/server/extension/sql/0.5.1/60_postalcodes.sql @@ -5,7 +5,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -36,7 +36,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -67,7 +67,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -98,7 +98,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/sql/0.5.1/70_ips.sql b/server/extension/sql/0.5.1/70_ips.sql index 5b70606..c3c5c63 100644 --- a/server/extension/sql/0.5.1/70_ips.sql +++ b/server/extension/sql/0.5.1/70_ips.sql @@ -5,7 +5,7 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) quota_service = QuotaService(user_geocoder_config, redis_conn) try: diff --git a/server/extension/test/0.5.0/expected/00_install_test.out b/server/extension/test/0.5.0/expected/00_install_test.out index c61b0a2..f4a1873 100644 --- a/server/extension/test/0.5.0/expected/00_install_test.out +++ b/server/extension/test/0.5.0/expected/00_install_test.out @@ -19,7 +19,19 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh (1 row) -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/0.5.1/sql/00_install_test.sql b/server/extension/test/0.5.1/sql/00_install_test.sql index 1a296de..142f8a3 100644 --- a/server/extension/test/0.5.1/sql/00_install_test.sql +++ b/server/extension/test/0.5.1/sql/00_install_test.sql @@ -12,8 +12,8 @@ CREATE EXTENSION cdb_dataservices_server; SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); -SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/var/log/postgresql/geocodings.log"}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d0670d7..75e4e6e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -69,8 +69,8 @@ class IsolinesRoutingConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, username, orgname) - db_config = ServicesDBConfig(db_conn) config = self.__get_user_config(username, orgname) + db_config = ServicesDBConfig(db_conn) filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} self.__parse_config(filtered_config, db_config) @@ -136,9 +136,11 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): - def __init__(self, redis_connection, username, orgname=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(InternalGeocoderConfig, self).__init__(redis_connection, username, orgname) + db_config = ServicesDBConfig(db_conn) + self._log_path = db_config.geocoder_log_path @property def service_type(self): @@ -156,6 +158,10 @@ class InternalGeocoderConfig(ServiceConfig): def geocoding_quota(self): return None + @property + def log_path(self): + return self._log_path + class GeocoderConfig(ServiceConfig): @@ -289,7 +295,7 @@ class GeocoderConfig(ServiceConfig): @property def cost_per_hit(self): - self._cost_per_hit + return self._cost_per_hit @property def log_path(self): diff --git a/server/lib/python/cartodb_services/path b/server/lib/python/cartodb_services/path deleted file mode 100644 index ce15f7f..0000000 --- a/server/lib/python/cartodb_services/path +++ /dev/null @@ -1,4 +0,0 @@ -{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.316618", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} -{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.318872", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} -{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.322009", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} -{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.323941", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 7aaebd3..cc530c6 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -48,4 +48,4 @@ def _plpy_execute_side_effect(*args, **kwargs): elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": return [{'conf': '{"routing_app_key": "app_key"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": - return [{'conf': '{"geocoder_log_path": "path"}'}] + return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] From 6fb891ec8652815f6b54b51ba4b381e0826603f1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 16 Mar 2016 18:05:48 +0100 Subject: [PATCH 09/10] Added headers for the generated files --- server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql | 3 +++ server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql | 3 +++ 2 files changed, 6 insertions(+) diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql index ca41e78..3298675 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql @@ -1,3 +1,6 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.5.1'" to load this file. \quit DROP FUNCTION IF EXISTS cdb_dataservices_server._get_redis_conf_v2(text); DROP TYPE IF EXISTS cdb_dataservices_server._redis_conf_params; diff --git a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql index a31e055..4355cf2 100644 --- a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql @@ -1,3 +1,6 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.5.0'" to load this file. \quit CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( sentinel_master_id text, redis_host text, From f0a49d5fbce2b6388f612961a223e93dccfd1089 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 17 Mar 2016 10:15:12 +0100 Subject: [PATCH 10/10] Deleted the used_credits field --- .../lib/python/cartodb_services/cartodb_services/metrics/log.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 29759e7..1617368 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -69,7 +69,6 @@ class GeocoderLogger(Logger): "real_rows": successful_rows, "success": data['success'], "successful_rows": successful_rows, - "used_credits": 0, "username": self._service_config.username, "organization": self._service_config.organization }