From e7c58b9a511ff3aee984f0f4ed4d13c5815a0726 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 27 Jan 2016 19:04:27 +0100 Subject: [PATCH 1/3] Google geocoder working --- server/extension/Makefile | 5 +- .../cdb_geocoder_server--0.1.0--0.2.0.sql | 65 ++ .../cdb_geocoder_server--0.2.0--0.1.0.sql | 38 ++ .../extension/cdb_geocoder_server--0.2.0.sql | 638 ++++++++++++++++++ server/extension/cdb_geocoder_server.control | 2 +- server/extension/sql/0.2.0/00_header.sql | 2 + .../extension/sql/0.2.0/10_redis_helper.sql | 57 ++ .../extension/sql/0.2.0/15_config_helper.sql | 25 + .../extension/sql/0.2.0/20_geocode_street.sql | 84 +++ server/extension/sql/0.2.0/30_admin0.sql | 37 + server/extension/sql/0.2.0/40_admin1.sql | 89 +++ server/extension/sql/0.2.0/50_namedplaces.sql | 121 ++++ server/extension/sql/0.2.0/60_postalcodes.sql | 162 +++++ server/extension/sql/0.2.0/70_ips.sql | 49 ++ .../sql/0.2.0/90_geocoder_server_user.sql | 15 + .../test/0.2.0/expected/00_install_test.out | 30 + .../test/0.2.0/expected/20_street_test.out | 12 + .../test/0.2.0/expected/30_admin0_test.out | 47 ++ .../test/0.2.0/expected/40_admin1_test.out | 81 +++ .../0.2.0/expected/50_namedplaces_test.out | 136 ++++ .../0.2.0/expected/60_postalcodes_test.out | 163 +++++ .../test/0.2.0/expected/70_ips_test.out | 40 ++ .../90_remove_geocoder_api_user_test.out | 5 + .../test/0.2.0/sql/00_install_test.sql | 24 + .../test/0.2.0/sql/20_street_test.sql | 7 + .../test/0.2.0/sql/30_admin0_test.sql | 32 + .../test/0.2.0/sql/40_admin1_test.sql | 48 ++ .../test/0.2.0/sql/50_namedplaces_test.sql | 72 ++ .../test/0.2.0/sql/60_postalcodes_test.sql | 117 ++++ .../extension/test/0.2.0/sql/70_ips_test.sql | 24 + .../sql/90_remove_geocoder_api_user_test.sql | 5 + .../cartodb_geocoder/quota_service.py | 3 - .../python/cartodb_geocoder/requirements.txt | 3 + .../google_geocoder/__init__.py | 0 .../google_geocoder/exceptions.py | 7 + .../google_geocoder/googlemapsgeocoder.py | 49 ++ server/lib/python/google_geocoder/setup.py | 41 ++ .../{heremapsexceptions.py => exceptions.py} | 5 - .../heremaps/heremaps/heremapsgeocoder.py | 43 +- 39 files changed, 2349 insertions(+), 34 deletions(-) create mode 100644 server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql create mode 100644 server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql create mode 100644 server/extension/cdb_geocoder_server--0.2.0.sql create mode 100644 server/extension/sql/0.2.0/00_header.sql create mode 100644 server/extension/sql/0.2.0/10_redis_helper.sql create mode 100644 server/extension/sql/0.2.0/15_config_helper.sql create mode 100644 server/extension/sql/0.2.0/20_geocode_street.sql create mode 100644 server/extension/sql/0.2.0/30_admin0.sql create mode 100644 server/extension/sql/0.2.0/40_admin1.sql create mode 100644 server/extension/sql/0.2.0/50_namedplaces.sql create mode 100644 server/extension/sql/0.2.0/60_postalcodes.sql create mode 100644 server/extension/sql/0.2.0/70_ips.sql create mode 100644 server/extension/sql/0.2.0/90_geocoder_server_user.sql create mode 100644 server/extension/test/0.2.0/expected/00_install_test.out create mode 100644 server/extension/test/0.2.0/expected/20_street_test.out create mode 100644 server/extension/test/0.2.0/expected/30_admin0_test.out create mode 100644 server/extension/test/0.2.0/expected/40_admin1_test.out create mode 100644 server/extension/test/0.2.0/expected/50_namedplaces_test.out create mode 100644 server/extension/test/0.2.0/expected/60_postalcodes_test.out create mode 100644 server/extension/test/0.2.0/expected/70_ips_test.out create mode 100644 server/extension/test/0.2.0/expected/90_remove_geocoder_api_user_test.out create mode 100644 server/extension/test/0.2.0/sql/00_install_test.sql create mode 100644 server/extension/test/0.2.0/sql/20_street_test.sql create mode 100644 server/extension/test/0.2.0/sql/30_admin0_test.sql create mode 100644 server/extension/test/0.2.0/sql/40_admin1_test.sql create mode 100644 server/extension/test/0.2.0/sql/50_namedplaces_test.sql create mode 100644 server/extension/test/0.2.0/sql/60_postalcodes_test.sql create mode 100644 server/extension/test/0.2.0/sql/70_ips_test.sql create mode 100644 server/extension/test/0.2.0/sql/90_remove_geocoder_api_user_test.sql create mode 100644 server/lib/python/google_geocoder/google_geocoder/__init__.py create mode 100644 server/lib/python/google_geocoder/google_geocoder/exceptions.py create mode 100644 server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py create mode 100644 server/lib/python/google_geocoder/setup.py rename server/lib/python/heremaps/heremaps/{heremapsexceptions.py => exceptions.py} (79%) diff --git a/server/extension/Makefile b/server/extension/Makefile index c3f5de8..f99d972 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -12,8 +12,11 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ cdb_geocoder_server--0.0.1.sql \ + cdb_geocoder_server--0.1.0.sql \ cdb_geocoder_server--0.1.0--0.0.1.sql \ - cdb_geocoder_server--0.0.1--0.1.0.sql + cdb_geocoder_server--0.0.1--0.1.0.sql \ + cdb_geocoder_server--0.2.0--0.1.0.sql \ + cdb_geocoder_server--0.1.0--0.2.0.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) diff --git a/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql b/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql new file mode 100644 index 0000000..d6f0cd3 --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql @@ -0,0 +1,65 @@ +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from google_geocoder import googlemapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql b/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql new file mode 100644 index 0000000..fb1b50e --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.error('Google geocoder is not available yet') + return None +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_geocoder_server--0.2.0.sql b/server/extension/cdb_geocoder_server--0.2.0.sql new file mode 100644 index 0000000..fb4b601 --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.2.0.sql @@ -0,0 +1,638 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from google_geocoder import googlemapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) +$$ LANGUAGE plpythonu; +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/cdb_geocoder_server.control b/server/extension/cdb_geocoder_server.control index 967b8d3..040ae10 100644 --- a/server/extension/cdb_geocoder_server.control +++ b/server/extension/cdb_geocoder_server.control @@ -1,6 +1,6 @@ # cdb geocoder server extension comment = 'CartoDB server geocoder extension' -default_version = '0.1.0' +default_version = '0.2.0' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_geocoder_server diff --git a/server/extension/sql/0.2.0/00_header.sql b/server/extension/sql/0.2.0/00_header.sql new file mode 100644 index 0000000..949e06e --- /dev/null +++ b/server/extension/sql/0.2.0/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit diff --git a/server/extension/sql/0.2.0/10_redis_helper.sql b/server/extension/sql/0.2.0/10_redis_helper.sql new file mode 100644 index 0000000..ef5047c --- /dev/null +++ b/server/extension/sql/0.2.0/10_redis_helper.sql @@ -0,0 +1,57 @@ +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.2.0/15_config_helper.sql b/server/extension/sql/0.2.0/15_config_helper.sql new file mode 100644 index 0000000..dd37953 --- /dev/null +++ b/server/extension/sql/0.2.0/15_config_helper.sql @@ -0,0 +1,25 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.2.0/20_geocode_street.sql b/server/extension/sql/0.2.0/20_geocode_street.sql new file mode 100644 index 0000000..2f3f394 --- /dev/null +++ b/server/extension/sql/0.2.0/20_geocode_street.sql @@ -0,0 +1,84 @@ +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from google_geocoder import googlemapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.2.0/30_admin0.sql b/server/extension/sql/0.2.0/30_admin0.sql new file mode 100644 index 0000000..96bbc43 --- /dev/null +++ b/server/extension/sql/0.2.0/30_admin0.sql @@ -0,0 +1,37 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.2.0/40_admin1.sql b/server/extension/sql/0.2.0/40_admin1.sql new file mode 100644 index 0000000..44a1953 --- /dev/null +++ b/server/extension/sql/0.2.0/40_admin1.sql @@ -0,0 +1,89 @@ +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.2.0/50_namedplaces.sql b/server/extension/sql/0.2.0/50_namedplaces.sql new file mode 100644 index 0000000..44069f2 --- /dev/null +++ b/server/extension/sql/0.2.0/50_namedplaces.sql @@ -0,0 +1,121 @@ +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.2.0/60_postalcodes.sql b/server/extension/sql/0.2.0/60_postalcodes.sql new file mode 100644 index 0000000..1a20379 --- /dev/null +++ b/server/extension/sql/0.2.0/60_postalcodes.sql @@ -0,0 +1,162 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.2.0/70_ips.sql b/server/extension/sql/0.2.0/70_ips.sql new file mode 100644 index 0000000..5480c2d --- /dev/null +++ b/server/extension/sql/0.2.0/70_ips.sql @@ -0,0 +1,49 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.2.0/90_geocoder_server_user.sql b/server/extension/sql/0.2.0/90_geocoder_server_user.sql new file mode 100644 index 0000000..3c2b354 --- /dev/null +++ b/server/extension/sql/0.2.0/90_geocoder_server_user.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/test/0.2.0/expected/00_install_test.out b/server/extension/test/0.2.0/expected/00_install_test.out new file mode 100644 index 0000000..b386bec --- /dev/null +++ b/server/extension/test/0.2.0/expected/00_install_test.out @@ -0,0 +1,30 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); + cdb_setuserquotainbytes +------------------------- + 0 +(1 row) + diff --git a/server/extension/test/0.2.0/expected/20_street_test.out b/server/extension/test/0.2.0/expected/20_street_test.out new file mode 100644 index 0000000..526cc00 --- /dev/null +++ b/server/extension/test/0.2.0/expected/20_street_test.out @@ -0,0 +1,12 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/30_admin0_test.out b/server/extension/test/0.2.0/expected/30_admin0_test.out new file mode 100644 index 0000000..52850c4 --- /dev/null +++ b/server/extension/test/0.2.0/expected/30_admin0_test.out @@ -0,0 +1,47 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/40_admin1_test.out b/server/extension/test/0.2.0/expected/40_admin1_test.out new file mode 100644 index 0000000..9897f34 --- /dev/null +++ b/server/extension/test/0.2.0/expected/40_admin1_test.out @@ -0,0 +1,81 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/50_namedplaces_test.out b/server/extension/test/0.2.0/expected/50_namedplaces_test.out new file mode 100644 index 0000000..1ac8d73 --- /dev/null +++ b/server/extension/test/0.2.0/expected/50_namedplaces_test.out @@ -0,0 +1,136 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/60_postalcodes_test.out b/server/extension/test/0.2.0/expected/60_postalcodes_test.out new file mode 100644 index 0000000..01aa003 --- /dev/null +++ b/server/extension/test/0.2.0/expected/60_postalcodes_test.out @@ -0,0 +1,163 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/70_ips_test.out b/server/extension/test/0.2.0/expected/70_ips_test.out new file mode 100644 index 0000000..2386200 --- /dev/null +++ b/server/extension/test/0.2.0/expected/70_ips_test.out @@ -0,0 +1,40 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +---------------------------------------------------- + 0101000020E61000003333333333334440AE47E17A14AE0D40 +(1 row) + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.2.0/expected/90_remove_geocoder_api_user_test.out b/server/extension/test/0.2.0/expected/90_remove_geocoder_api_user_test.out new file mode 100644 index 0000000..c53fcfc --- /dev/null +++ b/server/extension/test/0.2.0/expected/90_remove_geocoder_api_user_test.out @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/extension/test/0.2.0/sql/00_install_test.sql b/server/extension/test/0.2.0/sql/00_install_test.sql new file mode 100644 index 0000000..8feac25 --- /dev/null +++ b/server/extension/test/0.2.0/sql/00_install_test.sql @@ -0,0 +1,24 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.2.0/sql/20_street_test.sql b/server/extension/test/0.2.0/sql/20_street_test.sql new file mode 100644 index 0000000..7613f5b --- /dev/null +++ b/server/extension/test/0.2.0/sql/20_street_test.sql @@ -0,0 +1,7 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/30_admin0_test.sql b/server/extension/test/0.2.0/sql/30_admin0_test.sql new file mode 100644 index 0000000..3851d4f --- /dev/null +++ b/server/extension/test/0.2.0/sql/30_admin0_test.sql @@ -0,0 +1,32 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); + +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/40_admin1_test.sql b/server/extension/test/0.2.0/sql/40_admin1_test.sql new file mode 100644 index 0000000..d3080bf --- /dev/null +++ b/server/extension/test/0.2.0/sql/40_admin1_test.sql @@ -0,0 +1,48 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); + +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/50_namedplaces_test.sql b/server/extension/test/0.2.0/sql/50_namedplaces_test.sql new file mode 100644 index 0000000..47c304a --- /dev/null +++ b/server/extension/test/0.2.0/sql/50_namedplaces_test.sql @@ -0,0 +1,72 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); + +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); + +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/60_postalcodes_test.sql b/server/extension/test/0.2.0/sql/60_postalcodes_test.sql new file mode 100644 index 0000000..2a8192e --- /dev/null +++ b/server/extension/test/0.2.0/sql/60_postalcodes_test.sql @@ -0,0 +1,117 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/70_ips_test.sql b/server/extension/test/0.2.0/sql/70_ips_test.sql new file mode 100644 index 0000000..f9875f1 --- /dev/null +++ b/server/extension/test/0.2.0/sql/70_ips_test.sql @@ -0,0 +1,24 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.2.0/sql/90_remove_geocoder_api_user_test.sql b/server/extension/test/0.2.0/sql/90_remove_geocoder_api_user_test.sql new file mode 100644 index 0000000..4efb88e --- /dev/null +++ b/server/extension/test/0.2.0/sql/90_remove_geocoder_api_user_test.sql @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 2aa832d..19b4bcd 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -32,19 +32,16 @@ class QuotaService: self._user_service.increment_service_use( self._user_geocoder_config.service_type, "success_responses", amount=amount) - self.increment_total_geocoder_use(amount) def increment_empty_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "empty_responses", amount=amount) - self.increment_total_geocoder_use(amount) def increment_failed_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "fail_responses", amount=amount) - self.increment_total_geocoder_use(amount) def increment_total_geocoder_use(self, amount=1): self._user_service.increment_service_use( diff --git a/server/lib/python/cartodb_geocoder/requirements.txt b/server/lib/python/cartodb_geocoder/requirements.txt index 20b7203..04c1953 100644 --- a/server/lib/python/cartodb_geocoder/requirements.txt +++ b/server/lib/python/cartodb_geocoder/requirements.txt @@ -1,6 +1,9 @@ redis==2.10.5 # Dependency with incsv in the import python-dateutil==2.2 +heremaps==0.0.1 +googlegeocoder==0.0.1 +googlemaps # Test mock==1.3.0 diff --git a/server/lib/python/google_geocoder/google_geocoder/__init__.py b/server/lib/python/google_geocoder/google_geocoder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/google_geocoder/google_geocoder/exceptions.py b/server/lib/python/google_geocoder/google_geocoder/exceptions.py new file mode 100644 index 0000000..4452254 --- /dev/null +++ b/server/lib/python/google_geocoder/google_geocoder/exceptions.py @@ -0,0 +1,7 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + + +class MalformedResult(Exception): + def __str__(self): + return repr('Malformed result. The API might have changed.') diff --git a/server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py b/server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py new file mode 100644 index 0000000..de392ea --- /dev/null +++ b/server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py @@ -0,0 +1,49 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +import googlemaps + +from exceptions import MalformedResult + + +class Geocoder: + """A Google Maps Geocoder wrapper for python""" + + def __init__(self, client_id, client_secret): + self.client_id = self._clean_client_id(client_id) + self.client_secret = client_secret + self.geocoder = googlemaps.Client( + client_id=self.client_id, client_secret=self.client_secret) + + def geocode_address(self, searchtext, city=None, state=None, + country=None): + opt_params = self._build_optional_parameters(city, state, country) + results = self.geocoder.geocode(address=searchtext, components=opt_params) + if results: + return self._extract_lng_lat_from_result(results[0]) + else: + return [] + + def _extract_lng_lat_from_result(self, result): + try: + location = result['geometry']['location'] + longitude = location['lng'] + latitude = location['lat'] + return [longitude, latitude] + except KeyError: + raise MalformedResult() + + def _build_optional_parameters(self, city=None, state=None, + country=None): + optional_params = {} + if city: + optional_params['locality'] = city + if state: + optional_params['administrative_area'] = state + if country: + optional_params['country'] = country + return optional_params + + def _clean_client_id(self, client_id): + # Consistency with how the client_id is saved in metadata + return client_id.replace('client=', '') diff --git a/server/lib/python/google_geocoder/setup.py b/server/lib/python/google_geocoder/setup.py new file mode 100644 index 0000000..85d499a --- /dev/null +++ b/server/lib/python/google_geocoder/setup.py @@ -0,0 +1,41 @@ +""" +A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension + +See: +https://github.com/CartoDB/geocoder-api +""" + +from setuptools import setup, find_packages + +setup( + name='google-geocoder', + + version='0.0.1', + + description='A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension', + + url='https://github.com/CartoDB/geocoder-api', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 2 - Beta', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps api mapping tools', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest'], + } +) + diff --git a/server/lib/python/heremaps/heremaps/heremapsexceptions.py b/server/lib/python/heremaps/heremaps/exceptions.py similarity index 79% rename from server/lib/python/heremaps/heremaps/heremapsexceptions.py rename to server/lib/python/heremaps/heremaps/exceptions.py index cb10713..f4a3015 100644 --- a/server/lib/python/heremaps/heremaps/heremapsexceptions.py +++ b/server/lib/python/heremaps/heremaps/exceptions.py @@ -16,11 +16,6 @@ class NoGeocodingParams(Exception): return repr('No params for geocoding specified') -class EmptyGeocoderResponse(Exception): - def __str__(self): - return repr('The request could not be geocoded') - - class MalformedResult(Exception): def __str__(self): return repr('Result structure is malformed') diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index 9ebe1df..26b9d43 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -4,10 +4,10 @@ import json import requests -from heremaps.heremapsexceptions import BadGeocodingParams -from heremaps.heremapsexceptions import EmptyGeocoderResponse -from heremaps.heremapsexceptions import NoGeocodingParams -from heremaps.heremapsexceptions import MalformedResult +from heremaps.exception import BadGeocodingParams +from heremaps.exception import EmptyGeocoderResponse +from heremaps.exception import NoGeocodingParams +from heremaps.exception import MalformedResult class Geocoder: @@ -50,10 +50,6 @@ class Geocoder: 'strictlanguagemode' ] + ADDRESS_PARAMS - app_id = '' - app_code = '' - maxresults = '' - def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): self.app_id = app_id @@ -62,18 +58,26 @@ class Geocoder: self.gen = gen self.host = host - def geocode(self, params): + def geocode(self, **kwargs): + params = {} + for key, value in kwargs.iteritems(): + if value: + params[key] = value + if not params: + raise NoGeocodingParams() + return self.geocode(params) + + def _execute_geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): raise BadGeocodingParams(params) - response = self.perform_request(params) try: + response = self._perform_request(params) results = response['Response']['View'][0]['Result'] + return self._extract_lng_lat_from_result(results) except IndexError: - raise EmptyGeocoderResponse() + return [] - return results - - def perform_request(self, params): + def _perform_request(self, params): request_params = { 'app_id': self.app_id, 'app_code': self.app_code, @@ -87,16 +91,7 @@ class Geocoder: else: response.raise_for_status() - def geocode_address(self, **kwargs): - params = {} - for key, value in kwargs.iteritems(): - if value: - params[key] = value - if not params: - raise NoGeocodingParams() - return self.geocode(params) - - def extract_lng_lat_from_result(self, result): + def _extract_lng_lat_from_result(self, result): try: location = result['Location'] except KeyError: From 3960c134840e2f46a24dbfc421ceecc6714a1928 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 29 Jan 2016 11:54:50 +0100 Subject: [PATCH 2/3] Refactor python library to unify and rename as cdb_services --- server/extension/.gitignore | 1 + .../cdb_geocoder_server--0.1.0--0.2.0.sql | 77 ++- .../cdb_geocoder_server--0.2.0--0.1.0.sql | 57 ++ .../extension/cdb_geocoder_server--0.2.0.sql | 638 ------------------ .../extension/sql/0.2.0/10_redis_helper.sql | 6 +- .../extension/sql/0.2.0/15_config_helper.sql | 4 +- .../extension/sql/0.2.0/20_geocode_street.sql | 20 +- .../cartodb_services}/__init__.py | 0 .../cartodb_services/google/__init__.py | 1 + .../cartodb_services/google}/exceptions.py | 0 .../cartodb_services/google/geocoder.py} | 31 +- .../cartodb_services/here/__init__.py | 1 + .../cartodb_services/here/exceptions.py | 21 + .../cartodb_services/here/geocoder.py} | 19 +- .../cartodb_services/metrics/__init__.py | 3 + .../cartodb_services/metrics/config.py} | 0 .../cartodb_services/metrics/quota.py} | 4 +- .../cartodb_services/metrics/user.py} | 2 +- .../cartodb_services/tools/__init__.py | 1 + .../cartodb_services/tools/redis_tools.py} | 2 +- .../requirements.txt | 8 +- .../setup.py | 10 +- .../test/test_config.py} | 20 +- .../test/test_googlegeocoder.py | 119 ++++ .../test/test_helper.py | 0 .../test/test_heremapsgeocoder.py} | 84 ++- .../test/test_quota_service.py | 35 +- .../test/test_user_service.py | 12 +- .../google_geocoder/__init__.py | 0 .../google_geocoder/exceptions.py | 7 - server/lib/python/google_geocoder/setup.py | 41 -- .../lib/python/heremaps/heremaps/__init__.py | 0 .../heremaps/heremaps/tests/__init__.py | 0 server/lib/python/heremaps/requirements.txt | 4 - server/lib/python/heremaps/setup.py | 41 -- 35 files changed, 397 insertions(+), 872 deletions(-) delete mode 100644 server/extension/cdb_geocoder_server--0.2.0.sql rename server/lib/python/{cartodb_geocoder/cartodb_geocoder => cartodb_services/cartodb_services}/__init__.py (100%) create mode 100644 server/lib/python/cartodb_services/cartodb_services/google/__init__.py rename server/lib/python/{heremaps/heremaps => cartodb_services/cartodb_services/google}/exceptions.py (100%) rename server/lib/python/{google_geocoder/google_geocoder/googlemapsgeocoder.py => cartodb_services/cartodb_services/google/geocoder.py} (63%) create mode 100644 server/lib/python/cartodb_services/cartodb_services/here/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/here/exceptions.py rename server/lib/python/{heremaps/heremaps/heremapsgeocoder.py => cartodb_services/cartodb_services/here/geocoder.py} (87%) create mode 100644 server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py rename server/lib/python/{cartodb_geocoder/cartodb_geocoder/config_helper.py => cartodb_services/cartodb_services/metrics/config.py} (100%) rename server/lib/python/{cartodb_geocoder/cartodb_geocoder/quota_service.py => cartodb_services/cartodb_services/metrics/quota.py} (95%) rename server/lib/python/{cartodb_geocoder/cartodb_geocoder/user_service.py => cartodb_services/cartodb_services/metrics/user.py} (99%) create mode 100644 server/lib/python/cartodb_services/cartodb_services/tools/__init__.py rename server/lib/python/{cartodb_geocoder/cartodb_geocoder/redis_helper.py => cartodb_services/cartodb_services/tools/redis_tools.py} (97%) rename server/lib/python/{cartodb_geocoder => cartodb_services}/requirements.txt (54%) rename server/lib/python/{cartodb_geocoder => cartodb_services}/setup.py (78%) rename server/lib/python/{cartodb_geocoder/test/test_config_helper.py => cartodb_services/test/test_config.py} (72%) create mode 100644 server/lib/python/cartodb_services/test/test_googlegeocoder.py rename server/lib/python/{cartodb_geocoder => cartodb_services}/test/test_helper.py (100%) rename server/lib/python/{heremaps/heremaps/tests/heremapsgeocoder_tests.py => cartodb_services/test/test_heremapsgeocoder.py} (58%) rename server/lib/python/{cartodb_geocoder => cartodb_services}/test/test_quota_service.py (73%) rename server/lib/python/{cartodb_geocoder => cartodb_services}/test/test_user_service.py (91%) delete mode 100644 server/lib/python/google_geocoder/google_geocoder/__init__.py delete mode 100644 server/lib/python/google_geocoder/google_geocoder/exceptions.py delete mode 100644 server/lib/python/google_geocoder/setup.py delete mode 100644 server/lib/python/heremaps/heremaps/__init__.py delete mode 100644 server/lib/python/heremaps/heremaps/tests/__init__.py delete mode 100644 server/lib/python/heremaps/requirements.txt delete mode 100644 server/lib/python/heremaps/setup.py diff --git a/server/extension/.gitignore b/server/extension/.gitignore index ac24359..e0b7c49 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -3,3 +3,4 @@ regression.diffs regression.out cdb_geocoder_server--0.0.1.sql cdb_geocoder_server--0.1.0.sql +cdb_geocoder_server--0.2.0.sql diff --git a/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql b/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql index d6f0cd3..86178ed 100644 --- a/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql +++ b/server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql @@ -1,19 +1,76 @@ + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; + CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - from heremaps import heremapsgeocoder - from cartodb_geocoder import quota_service + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] # -- Check the quota - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reach the limit of your quota') try: - geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_geocoder_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) @@ -35,16 +92,16 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - from google_geocoder import googlemapsgeocoder - from cartodb_geocoder import quota_service + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + quota_service = QuotaService(user_geocoder_config, redis_conn) try: - geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_geocoder_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql b/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql index fb1b50e..cb47325 100644 --- a/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql +++ b/server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql @@ -1,3 +1,60 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; + CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from heremaps import heremapsgeocoder diff --git a/server/extension/cdb_geocoder_server--0.2.0.sql b/server/extension/cdb_geocoder_server--0.2.0.sql deleted file mode 100644 index fb4b601..0000000 --- a/server/extension/cdb_geocoder_server--0.2.0.sql +++ /dev/null @@ -1,638 +0,0 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit -CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( - sentinel_host text, - sentinel_port int, - sentinel_master_id text, - redis_db text, - timeout float -); - --- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) -RETURNS cdb_geocoder_server._redis_conf_params AS $$ - conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) - conf = plpy.execute(conf_query)[0]['conf'] - if conf is None: - plpy.error("There is no redis configuration defined") - else: - import json - params = json.loads(conf) - return { - "sentinel_host": params['sentinel_host'], - "sentinel_port": params['sentinel_port'], - "sentinel_master_id": params['sentinel_master_id'], - "timeout": params['timeout'], - "redis_db": params['redis_db'] - } -$$ LANGUAGE plpythonu; - --- Get the connection to redis from cache or create a new one -CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) -RETURNS boolean AS $$ - cache_key = "redis_connection_{0}".format(user_id) - if cache_key in GD: - return False - else: - from cartodb_geocoder import redis_helper - metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.timeout, c.redis_db - from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.timeout, c.redis_db - from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], - metadata_config_params['sentinel_port'], - metadata_config_params['sentinel_master_id'], - timeout=metadata_config_params['timeout'], - redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], - metrics_config_params['sentinel_port'], - metrics_config_params['sentinel_master_id'], - timeout=metrics_config_params['timeout'], - redis_db=metrics_config_params['redis_db']).redis_connection() - GD[cache_key] = { - 'redis_metadata_connection': redis_metadata_connection, - 'redis_metrics_connection': redis_metrics_connection, - } - return True -$$ LANGUAGE plpythonu; --- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_geocoder_config_{0}".format(username) - if cache_key in GD: - return False - else: - import json - from cartodb_geocoder import config_helper - plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] - if not heremaps_conf_json: - heremaps_app_id = None - heremaps_app_code = None - else: - heremaps_conf = json.loads(heremaps_conf_json) - heremaps_app_id = heremaps_conf['app_id'] - heremaps_app_code = heremaps_conf['app_code'] - geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... - GD[cache_key] = geocoder_config - return True -$$ LANGUAGE plpythonu; --- Geocodes a street address given a searchtext and a state and/or country -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - plpy.error('Requested geocoder is not available') - -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from heremaps import heremapsgeocoder - from cartodb_geocoder import quota_service - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - # -- Check the quota - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reach the limit of your quota') - - try: - geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - quota_service.increment_success_geocoder_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from google_geocoder import googlemapsgeocoder - from cartodb_geocoder import quota_service - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) - - try: - geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - quota_service.increment_success_geocoder_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) -$$ LANGUAGE plpythonu; --- Interface of the server extension - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_admin0_polygons') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - - plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') - return rv[0]["mypolygon"] -$$ LANGUAGE plpythonu; - - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT n.the_geom as geom INTO ret - FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x - FROM (SELECT country_name q) g) d - LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x - LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; - - RETURN ret; - END -$$ LANGUAGE plpgsql; --- Interfacess of the server extension - ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [admin1_name], 1) - - plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') - return rv[0]["mypolygon"] -$$ LANGUAGE plpythonu; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) - rv = plpy.execute(plan, [admin1_name, country_name], 1) - - plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') - return rv[0]["mypolygon"] -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT q, ( - SELECT the_geom - FROM global_province_polygons - WHERE d.c = ANY (synonyms) - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM ( - SELECT - trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q - ) d - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) - SELECT - geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_province_polygons - WHERE p.c = ANY (synonyms) - AND iso3 = p.i - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM p) n; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - --- Interfacess of the server extension - ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) - rv = plpy.execute(plan, [city_name], 1) - - plpy.debug('Returning from Returning from geocode_namedplace') - return rv[0]["mypoint"] -$$ LANGUAGE plpythonu; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) - rv = plpy.execute(plan, [city_name, country_name], 1) - - plpy.debug('Returning from Returning from geocode_namedplace') - return rv[0]["mypoint"] -$$ LANGUAGE plpythonu; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) - rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) - - plpy.debug('Returning from Returning from geocode_namedplace') - return rv[0]["mypoint"] -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), - next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), - best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH inputcountry AS ( - SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 - ), - p AS ( - SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), - best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - --- Interface of the server extension - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) -RETURNS Geometry AS $$ - plpy.debug('Entering _cdb_geocode_postalcode_point') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) - rv = plpy.execute(plan, [code], 1) - - plpy.debug('Returning from _cdb_geocode_postalcode_point') - return rv[0]["point"] -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - plpy.debug('Entering _cdb_geocode_postalcode_point') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - - plpy.debug('Returning from _cdb_geocode_postalcode_point') - return rv[0]["point"] -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) -RETURNS Geometry AS $$ - plpy.debug('Entering _cdb_geocode_postalcode_polygon') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) - rv = plpy.execute(plan, [code], 1) - - plpy.debug('Returning from _cdb_geocode_postalcode_polygon') - return rv[0]["polygon"] -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - plpy.debug('Entering _cdb_geocode_postalcode_point') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - - plpy.debug('Returning from _cdb_geocode_postalcode_point') - return rv[0]["polygon"] -$$ LANGUAGE plpythonu; - - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; --- Interface of the server extension - -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) -RETURNS Geometry AS $$ - plpy.debug('Entering _cdb_geocode_ipaddress_point') - plpy.debug('user = %s' % username) - - #--TODO: rate limiting check - #--TODO: quota check - - #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html - plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) - rv = plpy.execute(plan, [ip], 1) - - plpy.debug('Returning from _cdb_geocode_ipaddress_point') - return rv[0]["point"] -$$ LANGUAGE plpythonu; - - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - - new_ip INET; - BEGIN - BEGIN - IF family(ip::inet) = 6 THEN - new_ip := ip::inet; - ELSE - new_ip := ('::ffff:' || ip)::inet; - END IF; - EXCEPTION WHEN OTHERS THEN - SELECT NULL as geom INTO ret; - RETURN ret; - END; - - WITH - ips AS (SELECT ip s, new_ip net), - matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) - SELECT geom INTO ret - FROM matches; - RETURN ret; -END -$$ LANGUAGE plpgsql; -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT * - FROM pg_catalog.pg_user - WHERE usename = 'geocoder_api') THEN - - CREATE USER geocoder_api; - END IF; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; - GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; - GRANT USAGE ON SCHEMA public TO geocoder_api; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; -END$$; \ No newline at end of file diff --git a/server/extension/sql/0.2.0/10_redis_helper.sql b/server/extension/sql/0.2.0/10_redis_helper.sql index ef5047c..a9f6415 100644 --- a/server/extension/sql/0.2.0/10_redis_helper.sql +++ b/server/extension/sql/0.2.0/10_redis_helper.sql @@ -32,19 +32,19 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - from cartodb_geocoder import redis_helper + from cartodb_services.tools import RedisConnection metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, c.sentinel_master_id, c.timeout, c.redis_db from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, c.sentinel_master_id, c.timeout, c.redis_db from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], metadata_config_params['sentinel_port'], metadata_config_params['sentinel_master_id'], timeout=metadata_config_params['timeout'], redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], metrics_config_params['sentinel_port'], metrics_config_params['sentinel_master_id'], timeout=metrics_config_params['timeout'], diff --git a/server/extension/sql/0.2.0/15_config_helper.sql b/server/extension/sql/0.2.0/15_config_helper.sql index dd37953..c44b56e 100644 --- a/server/extension/sql/0.2.0/15_config_helper.sql +++ b/server/extension/sql/0.2.0/15_config_helper.sql @@ -6,7 +6,7 @@ RETURNS boolean AS $$ return False else: import json - from cartodb_geocoder import config_helper + from cartodb_services.quota import GeocoderConfig plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] @@ -17,7 +17,7 @@ RETURNS boolean AS $$ heremaps_conf = json.loads(heremaps_conf_json) heremaps_app_id = heremaps_conf['app_id'] heremaps_app_code = heremaps_conf['app_code'] - geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) # --Think about the security concerns with this kind of global cache, it should be only available # --for this user session but... GD[cache_key] = geocoder_config diff --git a/server/extension/sql/0.2.0/20_geocode_street.sql b/server/extension/sql/0.2.0/20_geocode_street.sql index 2f3f394..b05e168 100644 --- a/server/extension/sql/0.2.0/20_geocode_street.sql +++ b/server/extension/sql/0.2.0/20_geocode_street.sql @@ -19,20 +19,20 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - from heremaps import heremapsgeocoder - from cartodb_geocoder import quota_service + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.quota import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] # -- Check the quota - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reach the limit of your quota') try: - geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_geocoder_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) @@ -54,16 +54,16 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - from google_geocoder import googlemapsgeocoder - from cartodb_geocoder import quota_service + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.quota import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + quota_service = QuotaService(user_geocoder_config, redis_conn) try: - geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) - coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_geocoder_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py similarity index 100% rename from server/lib/python/cartodb_geocoder/cartodb_geocoder/__init__.py rename to server/lib/python/cartodb_services/cartodb_services/__init__.py diff --git a/server/lib/python/cartodb_services/cartodb_services/google/__init__.py b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py new file mode 100644 index 0000000..4e8cd6b --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/google/__init__.py @@ -0,0 +1 @@ +from geocoder import GoogleMapsGeocoder diff --git a/server/lib/python/heremaps/heremaps/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/google/exceptions.py similarity index 100% rename from server/lib/python/heremaps/heremaps/exceptions.py rename to server/lib/python/cartodb_services/cartodb_services/google/exceptions.py diff --git a/server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py similarity index 63% rename from server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py rename to server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index de392ea..ed2a701 100644 --- a/server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -6,7 +6,7 @@ import googlemaps from exceptions import MalformedResult -class Geocoder: +class GoogleMapsGeocoder: """A Google Maps Geocoder wrapper for python""" def __init__(self, client_id, client_secret): @@ -15,24 +15,25 @@ class Geocoder: self.geocoder = googlemaps.Client( client_id=self.client_id, client_secret=self.client_secret) - def geocode_address(self, searchtext, city=None, state=None, - country=None): - opt_params = self._build_optional_parameters(city, state, country) - results = self.geocoder.geocode(address=searchtext, components=opt_params) - if results: - return self._extract_lng_lat_from_result(results[0]) - else: - return [] - - def _extract_lng_lat_from_result(self, result): + def geocode(self, searchtext, city=None, state=None, + country=None): try: - location = result['geometry']['location'] - longitude = location['lng'] - latitude = location['lat'] - return [longitude, latitude] + opt_params = self._build_optional_parameters(city, state, country) + results = self.geocoder.geocode(address=searchtext, + components=opt_params) + if results: + return self._extract_lng_lat_from_result(results[0]) + else: + return [] except KeyError: raise MalformedResult() + def _extract_lng_lat_from_result(self, result): + location = result['geometry']['location'] + longitude = location['lng'] + latitude = location['lat'] + return [longitude, latitude] + def _build_optional_parameters(self, city=None, state=None, country=None): optional_params = {} diff --git a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py new file mode 100644 index 0000000..5d910f3 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py @@ -0,0 +1 @@ +from geocoder import HereMapsGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py new file mode 100644 index 0000000..f4a3015 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py @@ -0,0 +1,21 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- +import json + + +class BadGeocodingParams(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr('Bad geocoding params: ' + json.dumps(self.value)) + + +class NoGeocodingParams(Exception): + def __str__(self): + return repr('No params for geocoding specified') + + +class MalformedResult(Exception): + def __str__(self): + return repr('Result structure is malformed') diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py similarity index 87% rename from server/lib/python/heremaps/heremaps/heremapsgeocoder.py rename to server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 26b9d43..37bbf76 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -4,13 +4,10 @@ import json import requests -from heremaps.exception import BadGeocodingParams -from heremaps.exception import EmptyGeocoderResponse -from heremaps.exception import NoGeocodingParams -from heremaps.exception import MalformedResult +from exceptions import * -class Geocoder: +class HereMapsGeocoder: 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -65,17 +62,19 @@ class Geocoder: params[key] = value if not params: raise NoGeocodingParams() - return self.geocode(params) + return self._execute_geocode(params) def _execute_geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): raise BadGeocodingParams(params) try: response = self._perform_request(params) - results = response['Response']['View'][0]['Result'] + results = response['Response']['View'][0]['Result'][0] return self._extract_lng_lat_from_result(results) except IndexError: return [] + except KeyError: + raise MalformedResult() def _perform_request(self, params): request_params = { @@ -92,11 +91,7 @@ class Geocoder: response.raise_for_status() def _extract_lng_lat_from_result(self, result): - try: - location = result['Location'] - except KeyError: - raise MalformedResult() - + location = result['Location'] longitude = location['DisplayPosition']['Longitude'] latitude = location['DisplayPosition']['Latitude'] diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py new file mode 100644 index 0000000..7336e9b --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -0,0 +1,3 @@ +from config import GeocoderConfig, ConfigException +from quota import QuotaService +from user import UserMetricsService diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py similarity index 100% rename from server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py rename to server/lib/python/cartodb_services/cartodb_services/metrics/config.py diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py similarity index 95% rename from server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py rename to server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 19b4bcd..9a373cf 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,4 +1,4 @@ -import user_service +from user import UserMetricsService from datetime import date @@ -8,7 +8,7 @@ class QuotaService: def __init__(self, user_geocoder_config, redis_connection): self._user_geocoder_config = user_geocoder_config - self._user_service = user_service.UserService( + self._user_service = UserMetricsService( self._user_geocoder_config, redis_connection) def check_user_quota(self): diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py similarity index 99% rename from server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py rename to server/lib/python/cartodb_services/cartodb_services/metrics/user.py index aa5ca98..947f782 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -2,7 +2,7 @@ from datetime import date, timedelta from dateutil.relativedelta import relativedelta -class UserService: +class UserMetricsService: """ Class to manage all the user info """ SERVICE_GEOCODER_NOKIA = 'geocoder_here' diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py new file mode 100644 index 0000000..dc49770 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -0,0 +1 @@ +from redis_tools import RedisConnection diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py similarity index 97% rename from server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py rename to server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 0c686bf..4ca9009 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -1,7 +1,7 @@ from redis.sentinel import Sentinel -class RedisHelper: +class RedisConnection: REDIS_DEFAULT_USER_DB = 5 REDIS_DEFAULT_TIMEOUT = 2 #seconds diff --git a/server/lib/python/cartodb_geocoder/requirements.txt b/server/lib/python/cartodb_services/requirements.txt similarity index 54% rename from server/lib/python/cartodb_geocoder/requirements.txt rename to server/lib/python/cartodb_services/requirements.txt index 04c1953..17e41c8 100644 --- a/server/lib/python/cartodb_geocoder/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -1,11 +1,13 @@ redis==2.10.5 +hiredis==0.1.5 # Dependency with incsv in the import python-dateutil==2.2 -heremaps==0.0.1 -googlegeocoder==0.0.1 -googlemaps +googlemaps==2.4.2 +# Dependency for googlemaps package +requests<=2.9.1 # Test mock==1.3.0 mockredispy==2.9.0.11 nose==1.3.7 +requests-mock==0.7.0 diff --git a/server/lib/python/cartodb_geocoder/setup.py b/server/lib/python/cartodb_services/setup.py similarity index 78% rename from server/lib/python/cartodb_geocoder/setup.py rename to server/lib/python/cartodb_services/setup.py index 1f8242c..3d2448e 100644 --- a/server/lib/python/cartodb_geocoder/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -1,5 +1,5 @@ """ -CartoDB Geocoder Python Library +CartoDB Services Python Library See: https://github.com/CartoDB/geocoder-api @@ -8,11 +8,11 @@ https://github.com/CartoDB/geocoder-api from setuptools import setup, find_packages setup( - name='cartodb_geocoder', + name='cartodb_services', - version='0.0.1', + version='0.1.0', - description='CartoDB Geocoder Python Library', + description='CartoDB Services API Python Library', url='https://github.com/CartoDB/geocoder-api', @@ -29,7 +29,7 @@ setup( 'Programming Language :: Python :: 2.7', ], - keywords='maps api mapping tools geocoder', + keywords='maps api mapping tools geocoder routing', packages=find_packages(exclude=['contrib', 'docs', 'tests']), diff --git a/server/lib/python/cartodb_geocoder/test/test_config_helper.py b/server/lib/python/cartodb_services/test/test_config.py similarity index 72% rename from server/lib/python/cartodb_geocoder/test/test_config_helper.py rename to server/lib/python/cartodb_services/test/test_config.py index 72b7d21..7588427 100644 --- a/server/lib/python/cartodb_geocoder/test/test_config_helper.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -1,41 +1,41 @@ import test_helper -from cartodb_geocoder import config_helper +from cartodb_services.metrics import GeocoderConfig, ConfigException from unittest import TestCase from nose.tools import assert_raises from mockredis import MockRedis from datetime import datetime, timedelta -class TestConfigHelper(TestCase): +class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + geocoder_config = GeocoderConfig(self.redis_conn, 'test_user', None, 'nokia_id', 'nokia_cod') - assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 - assert geocoder_config.soft_geocoding_limit == False + assert geocoder_config.soft_geocoding_limit is False def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): yesterday = datetime.today() - timedelta(days=1) test_helper.build_redis_user_config(self.redis_conn, 'test_user') test_helper.build_redis_org_config(self.redis_conn, 'test_org', quota=200, end_date=yesterday) - geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + geocoder_config = GeocoderConfig(self.redis_conn, 'test_user', 'test_org', 'nokia_id', 'nokia_cod') - assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 - assert geocoder_config.soft_geocoding_limit == False + assert geocoder_config.soft_geocoding_limit is False assert geocoder_config.period_end_date.date() == yesterday.date() def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): test_helper.build_redis_user_config(self.redis_conn, 'test_user') - assert_raises(config_helper.ConfigException, - config_helper.GeocoderConfig, + assert_raises(ConfigException, + GeocoderConfig, self.redis_conn, 'test_user', None, None, None) diff --git a/server/lib/python/cartodb_services/test/test_googlegeocoder.py b/server/lib/python/cartodb_services/test/test_googlegeocoder.py new file mode 100644 index 0000000..a015272 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_googlegeocoder.py @@ -0,0 +1,119 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +import unittest +import requests_mock + +from cartodb_services.google import GoogleMapsGeocoder +from cartodb_services.google.exceptions import BadGeocodingParams +from cartodb_services.google.exceptions import NoGeocodingParams +from cartodb_services.google.exceptions import MalformedResult + +requests_mock.Mocker.TEST_PREFIX = 'test_' + + +@requests_mock.Mocker() +class GoogleGeocoderTestCase(unittest.TestCase): + GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json' + + EMPTY_RESPONSE = """{ + "results" : [], + "status" : "ZERO_RESULTS" + }""" + + GOOD_RESPONSE = """{ + "results" : [ + { + "address_components" : [ + { + "long_name" : "1600", + "short_name" : "1600", + "types" : [ "street_number" ] + }, + { + "long_name" : "Amphitheatre Pkwy", + "short_name" : "Amphitheatre Pkwy", + "types" : [ "route" ] + }, + { + "long_name" : "Mountain View", + "short_name" : "Mountain View", + "types" : [ "locality", "political" ] + }, + { + "long_name" : "Santa Clara County", + "short_name" : "Santa Clara County", + "types" : [ "administrative_area_level_2", "political" ] + }, + { + "long_name" : "California", + "short_name" : "CA", + "types" : [ "administrative_area_level_1", "political" ] + }, + { + "long_name" : "United States", + "short_name" : "US", + "types" : [ "country", "political" ] + }, + { + "long_name" : "94043", + "short_name" : "94043", + "types" : [ "postal_code" ] + } + ], + "formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA", + "geometry" : { + "location" : { + "lat" : 37.4224764, + "lng" : -122.0842499 + }, + "location_type" : "ROOFTOP", + "viewport" : { + "northeast" : { + "lat" : 37.4238253802915, + "lng" : -122.0829009197085 + }, + "southwest" : { + "lat" : 37.4211274197085, + "lng" : -122.0855988802915 + } + } + }, + "place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA", + "types" : [ "street_address" ] + } + ], + "status" : "OK" + }""" + + MALFORMED_RESPONSE = """{"manolo": "escobar"}""" + + def setUp(self): + self.geocoder = GoogleMapsGeocoder('dummy_client_id', + 'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E') + + def test_geocode_address_with_valid_params(self, req_mock): + req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL, + text=self.GOOD_RESPONSE) + response = self.geocoder.geocode( + searchtext='Calle Eloy Gonzalo 27', + city='Madrid', + country='España') + + self.assertEqual(response[0], -122.0842499) + self.assertEqual(response[1], 37.4224764) + + def test_geocode_address_empty_response(self, req_mock): + req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL, + text=self.EMPTY_RESPONSE) + result = self.geocoder.geocode(searchtext='lkajfñlasjfñ') + self.assertEqual(result, []) + + def test_geocode_with_malformed_result(self, req_mock): + req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL, + text=self.MALFORMED_RESPONSE) + with self.assertRaises(MalformedResult): + self.geocoder.geocode( + searchtext='Calle Eloy Gonzalo 27', + city='Madrid', + country='España') diff --git a/server/lib/python/cartodb_geocoder/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py similarity index 100% rename from server/lib/python/cartodb_geocoder/test/test_helper.py rename to server/lib/python/cartodb_services/test/test_helper.py diff --git a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py similarity index 58% rename from server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py rename to server/lib/python/cartodb_services/test/test_heremapsgeocoder.py index bca926f..fc3a0d8 100644 --- a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py +++ b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py @@ -2,25 +2,28 @@ # -*- coding: utf-8 -*- import unittest +import requests_mock -from heremaps import heremapsgeocoder -from heremaps.heremapsexceptions import BadGeocodingParams -from heremaps.heremapsexceptions import EmptyGeocoderResponse -from heremaps.heremapsexceptions import NoGeocodingParams -from heremaps.heremapsexceptions import MalformedResult +from cartodb_services.here import HereMapsGeocoder +from cartodb_services.here.exceptions import BadGeocodingParams +from cartodb_services.here.exceptions import NoGeocodingParams +from cartodb_services.here.exceptions import MalformedResult + +requests_mock.Mocker.TEST_PREFIX = 'test_' -class GeocoderTestCase(unittest.TestCase): - EMPTY_RESPONSE = { +@requests_mock.Mocker() +class HereMapsGeocoderTestCase(unittest.TestCase): + EMPTY_RESPONSE = """{ "Response": { "MetaInfo": { "Timestamp": "2015-11-04T16:31:57.273+0000" }, "View": [] } - } + }""" - GOOD_RESPONSE = { + GOOD_RESPONSE = """{ "Response": { "MetaInfo": { "Timestamp": "2015-11-04T16:30:32.187+0000" @@ -28,7 +31,7 @@ class GeocoderTestCase(unittest.TestCase): "View": [{ "_type": "SearchResultsViewType", "ViewId": 0, - "Result": [{ + "Result": { "Relevance": 0.89, "MatchLevel": "street", "MatchQuality": { @@ -57,7 +60,7 @@ class GeocoderTestCase(unittest.TestCase): } }, "Address": { - "Label": "Calle de Eloy Gonzalo, Madrid, España", + "Label": "Calle de Eloy Gonzalo, Madrid, Espana", "Country": "ESP", "State": "Comunidad de Madrid", "County": "Madrid", @@ -65,7 +68,7 @@ class GeocoderTestCase(unittest.TestCase): "District": "Trafalgar", "Street": "Calle de Eloy Gonzalo", "AdditionalData": [{ - "value": "España", + "value": "Espana", "key": "CountryName" }, { @@ -78,45 +81,52 @@ class GeocoderTestCase(unittest.TestCase): }] } } - }] + } }] } - } + }""" + + MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): - self.geocoder = heremapsgeocoder.Geocoder(None, None) + self.geocoder = HereMapsGeocoder(None, None) - def test_geocode_address_with_valid_params(self): - self.geocoder.perform_request = lambda x: self.GOOD_RESPONSE - response = self.geocoder.geocode_address( + def test_geocode_address_with_valid_params(self, req_mock): + req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, + text=self.GOOD_RESPONSE) + response = self.geocoder.geocode( searchtext='Calle Eloy Gonzalo 27', city='Madrid', country='España') - def test_geocode_address_with_invalid_params(self): + self.assertEqual(response[0], -3.70126) + self.assertEqual(response[1], 40.43433) + + def test_geocode_address_with_invalid_params(self, req_mock): + req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, + text=self.GOOD_RESPONSE) with self.assertRaises(BadGeocodingParams): - self.geocoder.geocode_address( + self.geocoder.geocode( searchtext='Calle Eloy Gonzalo 27', manolo='escobar') - def test_geocode_address_with_no_params(self): + def test_geocode_address_with_no_params(self, req_mock): + req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, + text=self.GOOD_RESPONSE) with self.assertRaises(NoGeocodingParams): - self.geocoder.geocode_address() + self.geocoder.geocode() - def test_geocode_address_empty_response(self): - self.geocoder.perform_request = lambda x: self.EMPTY_RESPONSE - with self.assertRaises(EmptyGeocoderResponse): - self.geocoder.geocode_address(searchtext='lkajfñlasjfñ') - - def test_extract_lng_lat_from_result(self): - result = self.GOOD_RESPONSE['Response']['View'][0]['Result'][0] - coordinates = self.geocoder.extract_lng_lat_from_result(result) - - self.assertEqual(coordinates[0], -3.70126) - self.assertEqual(coordinates[1], 40.43433) - - def test_extract_lng_lat_from_result_with_malformed_result(self): - result = {'manolo': 'escobar'} + def test_geocode_address_empty_response(self, req_mock): + req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, + text=self.EMPTY_RESPONSE) + result = self.geocoder.geocode(searchtext='lkajfñlasjfñ') + self.assertEqual(result, []) + def test_geocode_with_malformed_result(self, req_mock): + req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, + text=self.MALFORMED_RESPONSE) with self.assertRaises(MalformedResult): - self.geocoder.extract_lng_lat_from_result(result) + self.geocoder.geocode( + searchtext='Calle Eloy Gonzalo 27', + city='Madrid', + country='España') diff --git a/server/lib/python/cartodb_geocoder/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py similarity index 73% rename from server/lib/python/cartodb_geocoder/test/test_quota_service.py rename to server/lib/python/cartodb_services/test/test_quota_service.py index 7c71315..ef63c5a 100644 --- a/server/lib/python/cartodb_geocoder/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis -from cartodb_geocoder import quota_service -from cartodb_geocoder import config_helper +from cartodb_services.metrics import QuotaService +from cartodb_services.metrics import GeocoderConfig from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -14,10 +14,6 @@ class TestQuotaService(TestCase): # organization user # org::::YYYYMM:DD -# def increment_geocoder_uses(self, username, orgname=None, -# date=date.today(), service='geocoder_here', -# metric='success_responses', amount=20): - def setUp(self): self.redis_conn = MockRedis() @@ -68,28 +64,19 @@ class TestQuotaService(TestCase): def test_should_check_user_increment_and_quota_check_correctly(self): qs = self.__build_quota_service('test_user', quota=2) qs.increment_success_geocoder_use() - assert qs.check_user_quota() == True + assert qs.check_user_quota() is True qs.increment_success_geocoder_use(amount=2) - assert qs.check_user_quota() == False + assert qs.check_user_quota() is False month = date.today().strftime('%Y%m') - name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) - total_requests = self.redis_conn.zscore(name, date.today().day) - assert total_requests == 3 def test_should_check_org_increment_and_quota_check_correctly(self): qs = self.__build_quota_service('test_user', orgname='test_org', quota=2) qs.increment_success_geocoder_use() - assert qs.check_user_quota() == True + assert qs.check_user_quota() is True qs.increment_success_geocoder_use(amount=2) - assert qs.check_user_quota() == False + assert qs.check_user_quota() is False month = date.today().strftime('%Y%m') - org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month) - org_total_requests = self.redis_conn.zscore(org_name, date.today().day) - assert org_total_requests == 3 - user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) - user_total_requests = self.redis_conn.zscore(user_name, date.today().day) - assert user_total_requests == 3 def __build_quota_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, @@ -101,9 +88,9 @@ class TestQuotaService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = config_helper.GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') - return quota_service.QuotaService(geocoder_config, - redis_connection = self.redis_conn) + geocoder_config = GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return QuotaService(geocoder_config, + redis_connection = self.redis_conn) diff --git a/server/lib/python/cartodb_geocoder/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py similarity index 91% rename from server/lib/python/cartodb_geocoder/test/test_user_service.py rename to server/lib/python/cartodb_services/test/test_user_service.py index 4aeb1c0..8535ec3 100644 --- a/server/lib/python/cartodb_geocoder/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis -from cartodb_geocoder import user_service -from cartodb_geocoder import config_helper +from cartodb_services.metrics import UserMetricsService +from cartodb_services.metrics import GeocoderConfig from datetime import datetime, date from unittest import TestCase from nose.tools import assert_raises @@ -84,7 +84,7 @@ class TestUserService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = config_helper.GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') - return user_service.UserService(geocoder_config, self.redis_conn) + geocoder_config = GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return UserMetricsService(geocoder_config, self.redis_conn) diff --git a/server/lib/python/google_geocoder/google_geocoder/__init__.py b/server/lib/python/google_geocoder/google_geocoder/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/lib/python/google_geocoder/google_geocoder/exceptions.py b/server/lib/python/google_geocoder/google_geocoder/exceptions.py deleted file mode 100644 index 4452254..0000000 --- a/server/lib/python/google_geocoder/google_geocoder/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/local/bin/python -# -*- coding: utf-8 -*- - - -class MalformedResult(Exception): - def __str__(self): - return repr('Malformed result. The API might have changed.') diff --git a/server/lib/python/google_geocoder/setup.py b/server/lib/python/google_geocoder/setup.py deleted file mode 100644 index 85d499a..0000000 --- a/server/lib/python/google_geocoder/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension - -See: -https://github.com/CartoDB/geocoder-api -""" - -from setuptools import setup, find_packages - -setup( - name='google-geocoder', - - version='0.0.1', - - description='A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension', - - url='https://github.com/CartoDB/geocoder-api', - - author='Data Services Team - CartoDB', - author_email='dataservices@cartodb.com', - - license='MIT', - - classifiers=[ - 'Development Status :: 2 - Beta', - 'Intended Audience :: Mapping comunity', - 'Topic :: Maps :: Mapping Tools', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2.7', - ], - - keywords='maps api mapping tools', - - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - - extras_require={ - 'dev': ['unittest'], - 'test': ['unittest'], - } -) - diff --git a/server/lib/python/heremaps/heremaps/__init__.py b/server/lib/python/heremaps/heremaps/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/lib/python/heremaps/heremaps/tests/__init__.py b/server/lib/python/heremaps/heremaps/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/lib/python/heremaps/requirements.txt b/server/lib/python/heremaps/requirements.txt deleted file mode 100644 index 18860e2..0000000 --- a/server/lib/python/heremaps/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -requests==2.9.1 - -# Test -nose==1.3.7 diff --git a/server/lib/python/heremaps/setup.py b/server/lib/python/heremaps/setup.py deleted file mode 100644 index d11ae40..0000000 --- a/server/lib/python/heremaps/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -A Here Maps API Python wrapper - -See: -https://developer.here.com -https://github.com/CartoDB/geocoder-api -""" - -from setuptools import setup, find_packages - -setup( - name='heremaps', - - version='0.0.1', - - description='A Here Maps API Python wrapper', - - url='https://github.com/CartoDB/geocoder-api', - - author='Data Services Team - CartoDB', - author_email='dataservices@cartodb.com', - - license='MIT', - - classifiers=[ - 'Development Status :: 5 - Production', - 'Intended Audience :: Mapping comunity', - 'Topic :: Maps :: Mapping Tools', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2.7', - ], - - keywords='maps api mapping tools', - - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - - extras_require={ - 'dev': ['unittest'], - 'test': ['unittest'], - } -) From 1367c46957cb3ecf9dba428bf23dd5d621a1e669 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 4 Feb 2016 10:38:12 +0100 Subject: [PATCH 3/3] Added automatic generated files warning header --- client/sql/0.0.1/00_header.sql | 1 + client/sql/0.1.0/00_header.sql | 1 + server/extension/sql/0.0.1/00_header.sql | 1 + server/extension/sql/0.1.0/00_header.sql | 1 + server/extension/sql/0.2.0/00_header.sql | 1 + 5 files changed, 5 insertions(+) diff --git a/client/sql/0.0.1/00_header.sql b/client/sql/0.0.1/00_header.sql index a140c33..1779c9f 100644 --- a/client/sql/0.0.1/00_header.sql +++ b/client/sql/0.0.1/00_header.sql @@ -1,2 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit diff --git a/client/sql/0.1.0/00_header.sql b/client/sql/0.1.0/00_header.sql index a140c33..1779c9f 100644 --- a/client/sql/0.1.0/00_header.sql +++ b/client/sql/0.1.0/00_header.sql @@ -1,2 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit diff --git a/server/extension/sql/0.0.1/00_header.sql b/server/extension/sql/0.0.1/00_header.sql index 949e06e..4bef15e 100644 --- a/server/extension/sql/0.0.1/00_header.sql +++ b/server/extension/sql/0.0.1/00_header.sql @@ -1,2 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit diff --git a/server/extension/sql/0.1.0/00_header.sql b/server/extension/sql/0.1.0/00_header.sql index 949e06e..4bef15e 100644 --- a/server/extension/sql/0.1.0/00_header.sql +++ b/server/extension/sql/0.1.0/00_header.sql @@ -1,2 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit diff --git a/server/extension/sql/0.2.0/00_header.sql b/server/extension/sql/0.2.0/00_header.sql index 949e06e..4bef15e 100644 --- a/server/extension/sql/0.2.0/00_header.sql +++ b/server/extension/sql/0.2.0/00_header.sql @@ -1,2 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit