From 9ecabf91441dac5a35e836dce56881c272949aa5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 19 Jan 2016 18:14:05 +0100 Subject: [PATCH 01/40] Create either metrics and metadata redis connections --- .../extension/sql/0.0.1/10_redis_helper.sql | 2 +- .../extension/sql/0.1.0/10_redis_helper.sql | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 server/extension/sql/0.1.0/10_redis_helper.sql diff --git a/server/extension/sql/0.0.1/10_redis_helper.sql b/server/extension/sql/0.0.1/10_redis_helper.sql index a8841fa..501a0cc 100644 --- a/server/extension/sql/0.0.1/10_redis_helper.sql +++ b/server/extension/sql/0.0.1/10_redis_helper.sql @@ -41,4 +41,4 @@ RETURNS boolean AS $$ redis_db=config_params['redis_db']).redis_connection() GD[user_id] = {'redis_connection': redis_connection} return True -$$ LANGUAGE plpythonu; \ No newline at end of file +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/10_redis_helper.sql b/server/extension/sql/0.1.0/10_redis_helper.sql new file mode 100644 index 0000000..c94c508 --- /dev/null +++ b/server/extension/sql/0.1.0/10_redis_helper.sql @@ -0,0 +1,48 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + if user_id in GD and 'redis_connection' in GD[user_id]: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[user_id] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; From 6740ada16e75bf89b9484c6afc18de14b3e9f1e8 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 20 Jan 2016 16:02:48 +0100 Subject: [PATCH 02/40] Makefile create the upgrade sql files for the extension too --- server/extension/Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/server/extension/Makefile b/server/extension/Makefile index b7f4973..b711756 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -2,8 +2,20 @@ # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_geocoder_server EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +SED = sed -DATA = $(EXTENSION)--$(EXTVERSION).sql +UPGRADABLE = \ + 0.0.1 \ + $(END) + +UPGRADES = \ + $(shell echo $(UPGRADABLE) | \ + $(SED) 's/^/$(EXTENSION)--/' | \ + $(SED) 's/$$/--$(EXTVERSION).sql/' | \ + $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g') + +DATA = $(EXTENSION)--$(EXTVERSION).sql \ + $(UPGRADES) REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) @@ -12,13 +24,15 @@ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) - SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) $(DATA): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA) >> $@ +$(EXTENSION)--%--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql + cp $< $@ + all: $(DATA) # Only meant for development time, do not use once a version is released From a7371c0a484d75c743764d375fd636b89d87731a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 20 Jan 2016 16:03:30 +0100 Subject: [PATCH 03/40] geocoder api server extension points to the new version --- server/extension/cdb_geocoder_server.control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/cdb_geocoder_server.control b/server/extension/cdb_geocoder_server.control index 7ac080a..967b8d3 100644 --- a/server/extension/cdb_geocoder_server.control +++ b/server/extension/cdb_geocoder_server.control @@ -1,6 +1,6 @@ # cdb geocoder server extension comment = 'CartoDB server geocoder extension' -default_version = '0.0.1' +default_version = '0.1.0' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_geocoder_server From b2fa8739e2d8d7577e1dc6fea1a49008118ae361 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 20 Jan 2016 16:52:55 +0100 Subject: [PATCH 04/40] User config now is retrieved from redis --- .../extension/sql/0.1.0/10_redis_helper.sql | 6 +- .../cartodb_geocoder/config_helper.py | 180 ++++++++++-------- .../cartodb_geocoder/redis_helper.py | 35 ++-- 3 files changed, 129 insertions(+), 92 deletions(-) diff --git a/server/extension/sql/0.1.0/10_redis_helper.sql b/server/extension/sql/0.1.0/10_redis_helper.sql index c94c508..5d7a43e 100644 --- a/server/extension/sql/0.1.0/10_redis_helper.sql +++ b/server/extension/sql/0.1.0/10_redis_helper.sql @@ -1,4 +1,5 @@ -- Get the Redis configuration from the _conf table -- +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf(); CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text) RETURNS cdb_geocoder_server._redis_conf_params AS $$ conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) @@ -20,7 +21,8 @@ $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) RETURNS boolean AS $$ - if user_id in GD and 'redis_connection' in GD[user_id]: + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: return False else: from cartodb_geocoder import redis_helper @@ -40,7 +42,7 @@ RETURNS boolean AS $$ metrics_config_params['sentinel_master_id'], timeout=metrics_config_params['timeout'], redis_db=metrics_config_params['redis_db']).redis_connection() - GD[user_id] = { + GD[cache_key] = { 'redis_metadata_connection': redis_metadata_connection, 'redis_metrics_connection': redis_metrics_connection, } diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index a2b148f..5914fe9 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -1,104 +1,130 @@ import json +from dateutil.parser import parse as date_parse + class ConfigException(Exception): pass + class UserConfig: - USER_CONFIG_KEYS = ['is_organization', 'entity_name'] + USER_CONFIG_KEYS = ['is_organization', 'entity_name'] - def __init__(self, user_config_json, db_user_id = None): - config = json.loads(user_config_json) - filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) - self._user_id = self.__extract_uuid(db_user_id) + def __init__(self, user_config_json, db_user_id=None): + config = json.loads(user_config_json) + filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()} + self.__check_config(filtered_config) + self.__parse_config(filtered_config) - def __check_config(self, filtered_config): - if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): - raise ConfigException("Passed user configuration is not correct, check it please") + def __check_config(self, filtered_config): + if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): + raise ConfigException( + "Passed user configuration is not correct, check it please") - return True + return True - @property - def is_organization(self): - return self._is_organization + @property + def is_organization(self): + return self._is_organization - @property - def entity_name(self): - return self._entity_name + @property + def entity_name(self): + return self._entity_name - @property - def user_id(self): - return self._user_id + def __parse_config(self, filtered_config): + self._is_organization = filtered_config['is_organization'] + self._entity_name = filtered_config['entity_name'] - def __parse_config(self, filtered_config): - self._is_organization = filtered_config['is_organization'] - self._entity_name = filtered_config['entity_name'] - - def __extract_uuid(self, db_user_id): - # Format: development_cartodb_user_ - return db_user_id.split('_')[-1] class GeocoderConfig: - GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', - 'nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER = 'nokia' - NOKIA_QUOTA_KEY = 'nokia_monthly_quota' - NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit' - GOOGLE_GEOCODER = 'google' - GEOCODER_TYPE = 'street_geocoder_provider' - GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key' + GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', + 'geocoding_quota', 'soft_geocoding_limit', + 'geocoder_type', 'period_end_date'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] + NOKIA_GEOCODER = 'heremaps' + GOOGLE_GEOCODER = 'google' + GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' + GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' + GEOCODER_TYPE = 'geocoder_type' + QUOTA_KEY = 'geocoding_quota' + SOFT_LIMIT_KEY = 'soft_geocoding_limit' + PERIOD_END_DATE = 'period_end_date' - def __init__(self, geocoder_config_json): - config = json.loads(geocoder_config_json) - filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) + def __init__(self, redis_connection, username, orgname=None): + self._redis_connection = redis_connection + config = self.__get_user_config(username, orgname) + filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} + self.__check_config(filtered_config) + self.__parse_config(filtered_config) - def __check_config(self, filtered_config): - if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): - raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""") - elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: - if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): - raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") + def __get_user_config(self, username, orgname=None): + user_config = self._redis_connection.hgetall( + "rails:users:{0}".format(username)) + if orgname: + org_config = self._redis_connection.hgetall( + "rails:orgs:{0}".format(orgname)) + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] + user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] - return True + return user_config - def __parse_config(self, filtered_config): - self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() - self._google_maps_private_key = None - self._nokia_monthly_quota = 0 - self._nokia_soft_geocoder_limit = False - if self.GOOGLE_GEOCODER == self._geocoder_type: - self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] - elif self.NOKIA_GEOCODER == self._geocoder_type: - self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY] - self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY] + def __check_config(self, filtered_config): + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): + raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""") + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): + raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") - @property - def service_type(self): - return self._geocoder_type + return True - @property - def nokia_geocoder(self): - return self._geocoder_type == self.NOKIA_GEOCODER + def __parse_config(self, filtered_config): + self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + self._google_maps_private_key = None + self._nokia_monthly_quota = 0 + self._nokia_soft_geocoder_limit = False + if self.GOOGLE_GEOCODER == self._geocoder_type: + self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] + elif self.NOKIA_GEOCODER == self._geocoder_type: + self._geocoding_quota = filtered_config[self.QUOTA_KEY] + self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY] - @property - def google_geocoder(self): - return self._geocoder_type == self.GOOGLE_GEOCODER + @property + def service_type(self): + if self._geocoder_type == self.GOOGLE_GEOCODER: + return 'geocoder_google' + else: + return 'geocoder_here' - @property - def google_api_key(self): - return self._google_maps_private_key + @property + def heremaps_geocoder(self): + return self._geocoder_type == self.NOKIA_GEOCODER - @property - def nokia_monthly_quota(self): - return self._nokia_monthly_quota + @property + def google_geocoder(self): + return self._geocoder_type == self.GOOGLE_GEOCODER - @property - def nokia_soft_limit(self): - return self._nokia_soft_geocoder_limit \ No newline at end of file + @property + def google_client_id(self): + return self._google_maps_client_id + + @property + def google_api_key(self): + return self._google_maps_private_key + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py index 27d4800..0c686bf 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -1,20 +1,29 @@ from redis.sentinel import Sentinel + class RedisHelper: - REDIS_DEFAULT_USER_DB = 5 - REDIS_TIMEOUT = 2 #seconds + REDIS_DEFAULT_USER_DB = 5 + REDIS_DEFAULT_TIMEOUT = 2 #seconds + REDIS_SENTINEL_DEFAULT_PORT = 26379 - def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.sentinel_host = sentinel_host - self.sentinel_port = sentinel_port - self.sentinel_master_id = sentinel_master_id - self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT - self.redis_db = redis_db + def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, + redis_db=REDIS_DEFAULT_USER_DB, **kwargs): + self.sentinel_host = sentinel_host + self.sentinel_port = sentinel_port + self.sentinel_master_id = sentinel_master_id + self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT + self.redis_db = redis_db - def redis_connection(self): - return self.__create_redis_connection() + def redis_connection(self): + return self.__create_redis_connection() - def __create_redis_connection(self): - sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout) - return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db) \ No newline at end of file + def __create_redis_connection(self): + sentinel = Sentinel([(self.sentinel_host, + self.REDIS_SENTINEL_DEFAULT_PORT)], + socket_timeout=self.timeout) + return sentinel.master_for( + self.sentinel_master_id, + socket_timeout=self.timeout, + db=self.redis_db + ) From fc35911b91ff2ece8471227668b3e15d620201d9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 20 Jan 2016 20:57:24 +0100 Subject: [PATCH 05/40] Geocoder street function with quota checking --- server/extension/.gitignore | 2 + .../extension/sql/0.1.0/15_config_helper.sql | 16 +++ .../extension/sql/0.1.0/20_geocode_street.sql | 30 ++++ .../cartodb_geocoder/config_helper.py | 41 +----- .../cartodb_geocoder/quota_service.py | 54 +++++--- .../cartodb_geocoder/user_service.py | 128 +++++++++++------- .../python/cartodb_geocoder/requirements.txt | 5 +- 7 files changed, 175 insertions(+), 101 deletions(-) create mode 100644 server/extension/sql/0.1.0/15_config_helper.sql create mode 100644 server/extension/sql/0.1.0/20_geocode_street.sql diff --git a/server/extension/.gitignore b/server/extension/.gitignore index 7b6ce39..340ecff 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -2,4 +2,6 @@ results/ regression.diffs regression.out cdb_geocoder_server--0.0.1.sql +cdb_geocoder_server--0.1.0.sql +cdb_geocoder_server--0.0.1--0.1.0.sql diff --git a/server/extension/sql/0.1.0/15_config_helper.sql b/server/extension/sql/0.1.0/15_config_helper.sql new file mode 100644 index 0000000..3154318 --- /dev/null +++ b/server/extension/sql/0.1.0/15_config_helper.sql @@ -0,0 +1,16 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql new file mode 100644 index 0000000..32fcdbb --- /dev/null +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -0,0 +1,30 @@ +-- Geocodes a street address given a searchtext and a state and/or country +DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT); +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import json + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn, username, orgname) + if not quota_service.check_user_quota(): + plpy.error('You have reach limit of your quota') + + heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf']) + app_id = heremaps_conf['geocoder']['app_id'] + app_code = heremaps_conf['geocoder']['app_code'] + + geocoder = heremapsgeocoder.Geocoder(app_id, app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + + return point['st_setsrid'] +$$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 5914fe9..1eef3ba 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -6,36 +6,6 @@ class ConfigException(Exception): pass -class UserConfig: - - USER_CONFIG_KEYS = ['is_organization', 'entity_name'] - - def __init__(self, user_config_json, db_user_id=None): - config = json.loads(user_config_json) - filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()} - self.__check_config(filtered_config) - self.__parse_config(filtered_config) - - def __check_config(self, filtered_config): - if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): - raise ConfigException( - "Passed user configuration is not correct, check it please") - - return True - - @property - def is_organization(self): - return self._is_organization - - @property - def entity_name(self): - return self._entity_name - - def __parse_config(self, filtered_config): - self._is_organization = filtered_config['is_organization'] - self._entity_name = filtered_config['entity_name'] - - class GeocoderConfig: GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', @@ -85,14 +55,17 @@ class GeocoderConfig: self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) self._google_maps_private_key = None - self._nokia_monthly_quota = 0 - self._nokia_soft_geocoder_limit = False + self._geocoding_quota = 0 + self._soft_geocoding_limit = False if self.GOOGLE_GEOCODER == self._geocoder_type: self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] elif self.NOKIA_GEOCODER == self._geocoder_type: - self._geocoding_quota = filtered_config[self.QUOTA_KEY] - self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY] + self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) + if filtered_config[self.SOFT_LIMIT_KEY] == 'true': + self._soft_geocoding_limit = True + else: + self._soft_geocoding_limit = False @property def service_type(self): diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 001eb34..061ffb3 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -1,31 +1,51 @@ import user_service -import config_helper from datetime import date -class QuotaService: - """ Class to manage all the quota operation for the Geocoder SQL API Extension """ - def __init__(self, user_config, geocoder_config, redis_connection): - self._user_config = user_config - self._geocoder_config = geocoder_config - self._user_service = user_service.UserService(self._user_config, - self._geocoder_config.service_type, redis_connection) +class QuotaService: + """ Class to manage all the quota operation for + the Geocoder SQL API Extension """ + + def __init__(self, user_geocoder_config, redis_connection, username, orgname=None): + self._user_geocoder_config = user_geocoder_config + self._user_service = user_service.UserService( + self._user_geocoder_config, + redis_connection, + username, + orgname + ) def check_user_quota(self): """ Check if the current user quota surpasses the current quota """ # We don't have quota check for google geocoder - if self._geocoder_config.google_geocoder: + if self._user_geocoder_config.google_geocoder: return True - user_quota = self._geocoder_config.nokia_monthly_quota + user_quota = self._user_geocoder_config.geocoding_quota today = date.today() - service_type = self._geocoder_config.service_type - current_used = self._user_service.used_quota(service_type, today.year, today.month) - soft_geocoder_limit = self._geocoder_config.nokia_soft_limit + service_type = self._user_geocoder_config.service_type + current_used = self._user_service.used_quota(service_type, today) + soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit - print "User quota: {0} --- current_used: {1} --- limit: {2}".format(user_quota, current_used, soft_geocoder_limit) + print "User quota: {0} --- current_used: {1} --- limit: {2}".format( + user_quota, current_used, soft_geocoding_limit) - return True if soft_geocoder_limit or current_used <= user_quota else False + if soft_geocoding_limit or current_used <= user_quota: + return True + else: + return False - def increment_geocoder_use(self, amount=1): - self._user_service.increment_service_use(self._geocoder_config.service_type) \ No newline at end of file + def increment_successful_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "success_responses" + ) + + def increment_empty_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "empty_responses" + ) + + def increment_failed_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "failed_responses" + ) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index 142056e..cdb72dd 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -1,64 +1,96 @@ -import redis_helper -from datetime import date +from datetime import date, timedelta +from dateutil.relativedelta import relativedelta + class UserService: - """ Class to manage all the user info """ + """ Class to manage all the user info """ - GEOCODING_QUOTA_KEY = "geocoding_quota" - GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit" + SERVICE_GEOCODER_NOKIA = 'geocoder_here' + SERVICE_GEOCODER_GOOGLE = 'geocoder_google' + SERVICE_GEOCODER_CACHE = 'geocoder_cache' - REDIS_CONNECTION_KEY = "redis_connection" - REDIS_CONNECTION_HOST = "redis_host" - REDIS_CONNECTION_PORT = "redis_port" - REDIS_CONNECTION_DB = "redis_db" + GEOCODING_QUOTA_KEY = "geocoding_quota" + GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit" - def __init__(self, user_config, service_type, redis_connection): - self.user_config = user_config - self.service_type = service_type - self._redis_connection = redis_connection + REDIS_CONNECTION_KEY = "redis_connection" + REDIS_CONNECTION_HOST = "redis_host" + REDIS_CONNECTION_PORT = "redis_port" + REDIS_CONNECTION_DB = "redis_db" - def used_quota(self, service_type, year, month, day=None): - """ Recover the used quota for the user in the current month """ - redis_key_data = self.__get_redis_key(service_type, year, month, day) - current_use = self._redis_connection.hget(redis_key_data['redis_name'], redis_key_data['redis_key']) - return int(current_use) if current_use else 0 + def __init__(self, user_geocoder_config, redis_connection, username, orgname=None): + self._user_geocoder_config = user_geocoder_config + self._redis_connection = redis_connection + self._username = username + self._orgname = orgname - def increment_service_use(self, service_type, date=date.today(), amount=1): - """ Increment the services uses in monthly and daily basis""" - self.__increment_monthly_uses(date, service_type, amount) - self.__increment_daily_uses(date, service_type, amount) + def used_quota(self, service_type, date): + """ Recover the used quota for the user in the current month """ + date_from, date_to = self.__current_billing_cycle() + current_use = 0 + success_responses = self.__get_metrics(service_type, + 'success_responses', date_from, + date_to) + empty_responses = self.__get_metrics(service_type, + 'empty_responses', date_from, + date_to) + current_use += (success_responses + empty_responses) + if service_type == self.SERVICE_GEOCODER_NOKIA: + cache_hits = self.__get_metrics(self.SERVICE_GEOCODER_CACHE, + 'success_responses', date_from, + date_to) + current_use += cache_hits - # Private functions + return current_use - def __increment_monthly_uses(self, date, service_type, amount): - redis_key_data = self.__get_redis_key(service_type, date.year, date.month) - self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) + def increment_service_use(self, service_type, metric, date=date.today(), amount=1): + """ Increment the services uses in monthly and daily basis""" + self.__increment_user_uses(service_type, metric, date, amount) + if self._orgname: + self.__increment_organization_uses(service_type, metric, date, amount) - def __increment_daily_uses(self, date, service_type, amount): - redis_key_data = self.__get_redis_key(service_type, date.year, date.month, date.day) - self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) + # Private functions - def __get_redis_key(self, service_type, year, month, day=None): - redis_name = self.__parse_redis_name(service_type,day) - redis_key = self.__parse_redis_key(year,month,day) + def __increment_user_uses(self, service_type, metric, date, amount): + redis_prefix = self.__parse_redis_prefix("user", self._username, + service_type, metric, date) + self._redis_connection.hincrby(redis_prefix, date.day, amount) - return {'redis_name': redis_name, 'redis_key': redis_key} + def __increment_organization_uses(self, service_type, metric, date, amount): + redis_prefix = self.__parse_redis_prefix("org", self._orgname, + service_type, metric, date) + self._redis_connection.hincrby(redis_prefix, date.day, amount) - def __parse_redis_name(self,service_type, day=None): - prefix = "org" if self.user_config.is_organization else "user" - dated_key = "used_quota_day" if day else "used_quota_month" - redis_name = "{0}:{1}:{2}:{3}".format( - prefix, self.user_config.entity_name, service_type, dated_key - ) - if self.user_config.is_organization and day: - redis_name = "{0}:{1}".format(redis_name, self.user_config.user_id) + def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date): + yearmonth_key = date.strftime('%Y%m') + redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, + service_type, metric, + yearmonth_key) - return redis_name + return redis_name - def __parse_redis_key(self,year,month,day=None): - if day: - redis_key = "{0}_{1}_{2}".format(year,month,day) - else: - redis_key = "{0}_{1}".format(year,month) + def __get_metrics(self, service, metric, date_from, date_to): + aggregated_metric = 0 + key_prefix = "org" if self._orgname else "user" + entity_name = self._orgname if self._orgname else self._username + for date in self.__generate_date_range(date_from, date_to): + redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, + service, metric, date) + score = self._redis_connection.zscore(redis_prefix, date.day) + aggregated_metric += score if score else 0 + return aggregated_metric - return redis_key + def __current_billing_cycle(self): + """ Return the begining and end date for the current billing cycle """ + end_period_day = self._user_geocoder_config.period_end_date.day + today = date.today() + if end_period_day > today.day: + temp_date = today + relativedelta(months=-1) + date_from = date(temp_date.year, temp_date.month, end_period_day) + else: + date_from = date(today.year, today.month, end_period_day) + + return date_from, today + + def __generate_date_range(self, date_from, date_to): + for n in range(int((date_to - date_from).days)): + yield date_from + timedelta(n) diff --git a/server/lib/python/cartodb_geocoder/requirements.txt b/server/lib/python/cartodb_geocoder/requirements.txt index 784f064..3b94e05 100644 --- a/server/lib/python/cartodb_geocoder/requirements.txt +++ b/server/lib/python/cartodb_geocoder/requirements.txt @@ -1,6 +1,7 @@ -redis-py==2.10.5 +redis==2.10.5 +python-dateutil==2.4.2 # Test mock==1.3.0 mockredispy==2.9.0.11 -nose==1.3.7 \ No newline at end of file +nose==1.3.7 From ad9c16b4df708f68d59c61144b037781f60e14a6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 08:23:41 +0100 Subject: [PATCH 06/40] Nokia geocoder config moved to the config helper --- .../extension/sql/0.1.0/15_config_helper.sql | 11 +++++- .../extension/sql/0.1.0/20_geocode_street.sql | 9 ++--- .../cartodb_geocoder/config_helper.py | 36 +++++++++++++++---- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/server/extension/sql/0.1.0/15_config_helper.sql b/server/extension/sql/0.1.0/15_config_helper.sql index 3154318..dd37953 100644 --- a/server/extension/sql/0.1.0/15_config_helper.sql +++ b/server/extension/sql/0.1.0/15_config_helper.sql @@ -5,10 +5,19 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: + import json from cartodb_geocoder import config_helper plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname) + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) # --Think about the security concerns with this kind of global cache, it should be only available # --for this user session but... GD[cache_key] = geocoder_config diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 32fcdbb..3063c94 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -2,7 +2,6 @@ DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT); CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - import json from heremaps import heremapsgeocoder from cartodb_geocoder import quota_service @@ -14,13 +13,9 @@ RETURNS Geometry AS $$ # -- Check the quota quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn, username, orgname) if not quota_service.check_user_quota(): - plpy.error('You have reach limit of your quota') + plpy.error('You have reach the limit of your quota') - heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf']) - app_id = heremaps_conf['geocoder']['app_id'] - app_code = heremaps_conf['geocoder']['app_code'] - - geocoder = heremapsgeocoder.Geocoder(app_id, app_code) + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) coordinates = geocoder.extract_lng_lat_from_result(results[0]) plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 1eef3ba..9c86791 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -10,9 +10,13 @@ class GeocoderConfig: GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', - 'geocoder_type', 'period_end_date'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] + 'geocoder_type', 'period_end_date', + 'heremaps_app_id', 'heremaps_app_code'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit', + 'heremaps_app_id', 'heremaps_app_code'] NOKIA_GEOCODER = 'heremaps' + NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' + NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' @@ -21,16 +25,21 @@ class GeocoderConfig: SOFT_LIMIT_KEY = 'soft_geocoding_limit' PERIOD_END_DATE = 'period_end_date' - def __init__(self, redis_connection, username, orgname=None): + def __init__(self, redis_connection, username, orgname=None, + heremaps_app_id=None, heremaps_app_code=None): self._redis_connection = redis_connection - config = self.__get_user_config(username, orgname) + config = self.__get_user_config(username, orgname, heremaps_app_id, + heremaps_app_code) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} self.__check_config(filtered_config) self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname=None): + def __get_user_config(self, username, orgname=None, heremaps_app_id=None, + heremaps_app_code=None): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) + user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id + user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code if orgname: org_config = self._redis_connection.hgetall( "rails:orgs:{0}".format(orgname)) @@ -44,10 +53,13 @@ class GeocoderConfig: def __check_config(self, filtered_config): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): - raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""") + raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") + if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]: + raise ConfigException("""Nokia geocoder configuration is missing. Check it please""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): - raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") + raise ConfigException("""Google geocoder need the mandatory + parameter 'google_maps_private_key'""") return True @@ -62,6 +74,8 @@ class GeocoderConfig: self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] elif self.NOKIA_GEOCODER == self._geocoder_type: self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) + self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] + self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] if filtered_config[self.SOFT_LIMIT_KEY] == 'true': self._soft_geocoding_limit = True else: @@ -101,3 +115,11 @@ class GeocoderConfig: @property def period_end_date(self): return self._period_end_date + + @property + def heremaps_app_id(self): + return self._heremaps_app_id + + @property + def heremaps_app_code(self): + return self._heremaps_app_code From 7c3ab87b7887f6fda22f8470f8cb3b923350e6e4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 13:45:29 +0100 Subject: [PATCH 07/40] When geocoder API calls heremaps increments the usage metrics --- .../extension/sql/0.1.0/20_geocode_street.sql | 24 ++++++++--- .../cartodb_geocoder/quota_service.py | 12 +++++- .../cartodb_geocoder/user_service.py | 4 +- .../heremaps/heremaps/heremapsexceptions.py | 12 ++++-- .../heremaps/heremaps/heremapsgeocoder.py | 41 +++++++++---------- server/lib/python/heremaps/requirements.txt | 1 + 6 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 server/lib/python/heremaps/requirements.txt diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 3063c94..3466564 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -15,11 +15,23 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): plpy.error('You have reach the limit of your quota') - geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) - coordinates = geocoder.extract_lng_lat_from_result(results[0]) - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + except heremapsgeocoder.EmptyGeocoderResponse: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) - return point['st_setsrid'] $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 061ffb3..a2dadc8 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -35,17 +35,25 @@ class QuotaService: else: return False - def increment_successful_geocoder_use(self, amount=1): + def increment_success_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "success_responses" ) + self.increment_total_geocoder_use(amount) def increment_empty_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "empty_responses" ) + self.increment_total_geocoder_use(amount) def increment_failed_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "failed_responses" + self._user_geocoder_config.service_type, "fail_responses" + ) + self.increment_total_geocoder_use(amount) + + def increment_total_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "total_requests" ) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index cdb72dd..fea0ab3 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -53,12 +53,12 @@ class UserService: def __increment_user_uses(self, service_type, metric, date, amount): redis_prefix = self.__parse_redis_prefix("user", self._username, service_type, metric, date) - self._redis_connection.hincrby(redis_prefix, date.day, amount) + self._redis_connection.zincrby(redis_prefix, date.day, amount) def __increment_organization_uses(self, service_type, metric, date, amount): redis_prefix = self.__parse_redis_prefix("org", self._orgname, service_type, metric, date) - self._redis_connection.hincrby(redis_prefix, date.day, amount) + self._redis_connection.zincrby(redis_prefix, date.day, amount) def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date): yearmonth_key = date.strftime('%Y%m') diff --git a/server/lib/python/heremaps/heremaps/heremapsexceptions.py b/server/lib/python/heremaps/heremaps/heremapsexceptions.py index 6416872..cb10713 100644 --- a/server/lib/python/heremaps/heremaps/heremapsexceptions.py +++ b/server/lib/python/heremaps/heremaps/heremapsexceptions.py @@ -1,22 +1,26 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- - import json + class BadGeocodingParams(Exception): def __init__(self, value): self.value = value + def __str__(self): - return repr('Bad geocoding params: ' + json.dumps(value)) + return repr('Bad geocoding params: ' + json.dumps(self.value)) + class NoGeocodingParams(Exception): def __str__(self): return repr('No params for geocoding specified') + class EmptyGeocoderResponse(Exception): def __str__(self): return repr('The request could not be geocoded') + class MalformedResult(Exception): - def __str__(self): - return repr('Result structure is malformed') + def __str__(self): + return repr('Result structure is malformed') diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index 696c79f..dd4bdb8 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -2,17 +2,19 @@ # -*- coding: utf-8 -*- import json -import urllib +import requests from heremaps.heremapsexceptions import BadGeocodingParams from heremaps.heremapsexceptions import EmptyGeocoderResponse from heremaps.heremapsexceptions import NoGeocodingParams from heremaps.heremapsexceptions import MalformedResult + class Geocoder: 'A Here Maps Geocoder wrapper for python' URL_GEOCODE_JSON = 'http://geocoder.api.here.com/6.2/geocode.json' + URL_DEV_GEOCODE_JSON = 'http://localhost:6083/geocode.json' DEFAULT_MAXRESULTS = 1 DEFAULT_GEN = 9 @@ -52,7 +54,8 @@ class Geocoder: app_code = '' maxresults = '' - def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, gen=DEFAULT_GEN): + def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, + gen=DEFAULT_GEN): self.app_id = app_id self.app_code = app_code self.maxresults = maxresults @@ -61,9 +64,7 @@ class Geocoder: def geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): raise BadGeocodingParams(params) - response = self.perform_request(params) - try: results = response['Response']['View'][0]['Result'] except IndexError: @@ -73,29 +74,25 @@ class Geocoder: def perform_request(self, params): request_params = { - 'app_id' : self.app_id, - 'app_code' : self.app_code, - 'maxresults' : self.maxresults, - 'gen' : self.gen - } + 'app_id': self.app_id, + 'app_code': self.app_code, + 'maxresults': self.maxresults, + 'gen': self.gen + } request_params.update(params) - - encoded_request_params = urllib.urlencode(request_params) - - response = json.load( - urllib.urlopen(self.URL_GEOCODE_JSON - + '?' - + encoded_request_params)) - - return response + response = requests.get(self.URL_DEV_GEOCODE_JSON, params=request_params) + if response.status_code == requests.codes.ok: + return json.loads(response.text) + else: + response.raise_for_status() def geocode_address(self, **kwargs): params = {} for key, value in kwargs.iteritems(): - if value: params[key] = value - - if not params: raise NoGeocodingParams() - + if value: + params[key] = value + if not params: + raise NoGeocodingParams() return self.geocode(params) def extract_lng_lat_from_result(self, result): diff --git a/server/lib/python/heremaps/requirements.txt b/server/lib/python/heremaps/requirements.txt new file mode 100644 index 0000000..ca0dee4 --- /dev/null +++ b/server/lib/python/heremaps/requirements.txt @@ -0,0 +1 @@ +requests==2.9.1 From 4d0cabb4290aae1e54c0d74619dc2143eddaeab0 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:03:23 +0100 Subject: [PATCH 08/40] Add username and orgname to the config object instead of pass them to every object --- .../extension/sql/0.1.0/20_geocode_street.sql | 2 +- .../cartodb_geocoder/config_helper.py | 20 ++++++++++++++++++- .../cartodb_geocoder/quota_service.py | 8 ++------ .../cartodb_geocoder/user_service.py | 6 +++--- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 3466564..941743e 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -11,7 +11,7 @@ RETURNS Geometry AS $$ user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] # -- Check the quota - quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn, username, orgname) + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reach the limit of your quota') diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 9c86791..993852a 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -11,7 +11,8 @@ class GeocoderConfig: GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', 'geocoder_type', 'period_end_date', - 'heremaps_app_id', 'heremaps_app_code'] + 'heremaps_app_id', 'heremaps_app_code', 'username', + 'orgname'] NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit', 'heremaps_app_id', 'heremaps_app_code'] NOKIA_GEOCODER = 'heremaps' @@ -23,6 +24,8 @@ class GeocoderConfig: GEOCODER_TYPE = 'geocoder_type' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' + USERNAME_KEY = 'username' + ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, username, orgname=None, @@ -38,6 +41,8 @@ class GeocoderConfig: heremaps_app_code=None): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) + user_config[self.USERNAME_KEY] = username + user_config[self.ORGNAME_KEY] = orgname user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code if orgname: @@ -64,6 +69,11 @@ class GeocoderConfig: return True def __parse_config(self, filtered_config): + self._username = filtered_config[self.USERNAME_KEY].lower() + if filtered_config[self.ORGNAME_KEY]: + self._orgname = filtered_config[self.ORGNAME_KEY].lower() + else: + self._orgname = None self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) self._google_maps_private_key = None @@ -123,3 +133,11 @@ class GeocoderConfig: @property def heremaps_app_code(self): return self._heremaps_app_code + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._orgname diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index a2dadc8..cdcdce4 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -6,14 +6,10 @@ class QuotaService: """ Class to manage all the quota operation for the Geocoder SQL API Extension """ - def __init__(self, user_geocoder_config, redis_connection, username, orgname=None): + def __init__(self, user_geocoder_config, redis_connection): self._user_geocoder_config = user_geocoder_config self._user_service = user_service.UserService( - self._user_geocoder_config, - redis_connection, - username, - orgname - ) + self._user_geocoder_config, redis_connection) def check_user_quota(self): """ Check if the current user quota surpasses the current quota """ diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index fea0ab3..461d70f 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -17,11 +17,11 @@ class UserService: REDIS_CONNECTION_PORT = "redis_port" REDIS_CONNECTION_DB = "redis_db" - def __init__(self, user_geocoder_config, redis_connection, username, orgname=None): + def __init__(self, user_geocoder_config, redis_connection): self._user_geocoder_config = user_geocoder_config self._redis_connection = redis_connection - self._username = username - self._orgname = orgname + self._username = user_geocoder_config.username + self._orgname = user_geocoder_config.organization def used_quota(self, service_type, date): """ Recover the used quota for the user in the current month """ From af051e2ed58ee86323d5069527ce13e09d7ddd80 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:04:08 +0100 Subject: [PATCH 09/40] Set the soft geocoding limit always --- .../cartodb_geocoder/cartodb_geocoder/config_helper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 993852a..dc158bf 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -86,10 +86,11 @@ class GeocoderConfig: self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] - if filtered_config[self.SOFT_LIMIT_KEY] == 'true': - self._soft_geocoding_limit = True - else: - self._soft_geocoding_limit = False + + if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_geocoding_limit = True + else: + self._soft_geocoding_limit = False @property def service_type(self): From 111804dafc5d23421da3e8a5aa1cfdfd23eb33e1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:05:08 +0100 Subject: [PATCH 10/40] Refactor of the google geocoder keys in the config object --- .../cartodb_geocoder/cartodb_geocoder/config_helper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index dc158bf..def1b47 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -76,11 +76,12 @@ class GeocoderConfig: self._orgname = None self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) - self._google_maps_private_key = None + self._google_maps_api_key = None + self._google_maps_client_id = None self._geocoding_quota = 0 self._soft_geocoding_limit = False if self.GOOGLE_GEOCODER == self._geocoder_type: - self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] elif self.NOKIA_GEOCODER == self._geocoder_type: self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) @@ -113,7 +114,7 @@ class GeocoderConfig: @property def google_api_key(self): - return self._google_maps_private_key + return self._google_maps_api_key @property def geocoding_quota(self): From 3488d3f9905f162b6f09152ce966bbfd61eb4168 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:05:34 +0100 Subject: [PATCH 11/40] Pass the amount always --- .../cartodb_geocoder/quota_service.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index cdcdce4..5845a51 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -33,23 +33,23 @@ class QuotaService: def increment_success_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "success_responses" - ) + self._user_geocoder_config.service_type, "success_responses", + amount=amount) self.increment_total_geocoder_use(amount) def increment_empty_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "empty_responses" - ) + self._user_geocoder_config.service_type, "empty_responses", + amount=amount) self.increment_total_geocoder_use(amount) def increment_failed_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "fail_responses" - ) + self._user_geocoder_config.service_type, "fail_responses", + amount=amount) self.increment_total_geocoder_use(amount) def increment_total_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "total_requests" - ) + self._user_geocoder_config.service_type, "total_requests", + amount=amount) From 8a515580683c3d5621bd7ac0df0e2fd3c8763adb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:06:04 +0100 Subject: [PATCH 12/40] We need to have one date at least to check the quota --- .../python/cartodb_geocoder/cartodb_geocoder/user_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index 461d70f..f9b90bd 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -92,5 +92,5 @@ class UserService: return date_from, today def __generate_date_range(self, date_from, date_to): - for n in range(int((date_to - date_from).days)): + for n in range(int((date_to - date_from).days + 1)): yield date_from + timedelta(n) From aced68096b163939e8077daa1240c4de99549f46 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:07:07 +0100 Subject: [PATCH 13/40] Heremaps tests fixed --- .../heremaps/tests/heremapsgeocoder_tests.py | 96 +++++++++---------- server/lib/python/heremaps/requirements.txt | 3 + 2 files changed, 49 insertions(+), 50 deletions(-) diff --git a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py b/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py index 6fbd1c0..bca926f 100644 --- a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py +++ b/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py @@ -9,73 +9,72 @@ from heremaps.heremapsexceptions import EmptyGeocoderResponse from heremaps.heremapsexceptions import NoGeocodingParams from heremaps.heremapsexceptions import MalformedResult -from secrets import * class GeocoderTestCase(unittest.TestCase): EMPTY_RESPONSE = { - "Response":{ - "MetaInfo":{ - "Timestamp":"2015-11-04T16:31:57.273+0000" + "Response": { + "MetaInfo": { + "Timestamp": "2015-11-04T16:31:57.273+0000" }, - "View":[] + "View": [] } } GOOD_RESPONSE = { "Response": { "MetaInfo": { - "Timestamp":"2015-11-04T16:30:32.187+0000" + "Timestamp": "2015-11-04T16:30:32.187+0000" }, - "View":[{ - "_type":"SearchResultsViewType", - "ViewId":0, - "Result":[{ - "Relevance":0.89, - "MatchLevel":"street", - "MatchQuality":{ - "City":1.0, - "Street":[1.0] + "View": [{ + "_type": "SearchResultsViewType", + "ViewId": 0, + "Result": [{ + "Relevance": 0.89, + "MatchLevel": "street", + "MatchQuality": { + "City": 1.0, + "Street": [1.0] }, - "Location":{ - "LocationId":"NT_yyKB4r3mCWAX4voWgxPcuA", - "LocationType":"address", - "DisplayPosition":{ - "Latitude":40.43433, - "Longitude":-3.70126 + "Location": { + "LocationId": "NT_yyKB4r3mCWAX4voWgxPcuA", + "LocationType": "address", + "DisplayPosition": { + "Latitude": 40.43433, + "Longitude": -3.70126 }, - "NavigationPosition":[{ - "Latitude":40.43433, - "Longitude":-3.70126 + "NavigationPosition": [{ + "Latitude": 40.43433, + "Longitude": -3.70126 }], - "MapView":{ - "TopLeft":{ - "Latitude":40.43493, - "Longitude":-3.70404 + "MapView": { + "TopLeft": { + "Latitude": 40.43493, + "Longitude": -3.70404 }, - "BottomRight":{ - "Latitude":40.43373, - "Longitude":-3.69873 + "BottomRight": { + "Latitude": 40.43373, + "Longitude": -3.69873 } }, - "Address":{ - "Label":"Calle de Eloy Gonzalo, Madrid, España", - "Country":"ESP", - "State":"Comunidad de Madrid", - "County":"Madrid", - "City":"Madrid", - "District":"Trafalgar", - "Street":"Calle de Eloy Gonzalo", - "AdditionalData":[{ - "value":"España", - "key":"CountryName" + "Address": { + "Label": "Calle de Eloy Gonzalo, Madrid, España", + "Country": "ESP", + "State": "Comunidad de Madrid", + "County": "Madrid", + "City": "Madrid", + "District": "Trafalgar", + "Street": "Calle de Eloy Gonzalo", + "AdditionalData": [{ + "value": "España", + "key": "CountryName" }, { - "value":"Comunidad de Madrid", - "key":"StateName" + "value": "Comunidad de Madrid", + "key": "StateName" }, { - "value":"Madrid", - "key":"CountyName" + "value": "Madrid", + "key": "CountyName" }] } } @@ -117,10 +116,7 @@ class GeocoderTestCase(unittest.TestCase): self.assertEqual(coordinates[1], 40.43433) def test_extract_lng_lat_from_result_with_malformed_result(self): - result = {'manolo':'escobar'} + result = {'manolo': 'escobar'} with self.assertRaises(MalformedResult): self.geocoder.extract_lng_lat_from_result(result) - -if __name__ == '__main__': - main() diff --git a/server/lib/python/heremaps/requirements.txt b/server/lib/python/heremaps/requirements.txt index ca0dee4..18860e2 100644 --- a/server/lib/python/heremaps/requirements.txt +++ b/server/lib/python/heremaps/requirements.txt @@ -1 +1,4 @@ requests==2.9.1 + +# Test +nose==1.3.7 From c74947b0a5340f5f81981d8f3e2696d6baf8952f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jan 2016 18:07:27 +0100 Subject: [PATCH 14/40] Cartodb geocoder python module tests fixed --- .../cartodb_geocoder/quota_service.py | 3 - .../cartodb_geocoder/user_service.py | 38 ++--- .../test/test_config_helper.py | 69 ++++---- .../cartodb_geocoder/test/test_helper.py | 33 ++++ .../test/test_quota_service.py | 148 ++++++++++-------- .../test/test_user_service.py | 109 ++++++++----- 6 files changed, 237 insertions(+), 163 deletions(-) create mode 100644 server/lib/python/cartodb_geocoder/test/test_helper.py diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 5845a51..2aa832d 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -23,9 +23,6 @@ class QuotaService: current_used = self._user_service.used_quota(service_type, today) soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit - print "User quota: {0} --- current_used: {1} --- limit: {2}".format( - user_quota, current_used, soft_geocoding_limit) - if soft_geocoding_limit or current_used <= user_quota: return True else: diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index f9b90bd..aa5ca98 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -27,17 +27,17 @@ class UserService: """ Recover the used quota for the user in the current month """ date_from, date_to = self.__current_billing_cycle() current_use = 0 - success_responses = self.__get_metrics(service_type, - 'success_responses', date_from, - date_to) - empty_responses = self.__get_metrics(service_type, - 'empty_responses', date_from, + success_responses = self.get_metrics(service_type, + 'success_responses', date_from, date_to) + empty_responses = self.get_metrics(service_type, + 'empty_responses', date_from, + date_to) current_use += (success_responses + empty_responses) if service_type == self.SERVICE_GEOCODER_NOKIA: - cache_hits = self.__get_metrics(self.SERVICE_GEOCODER_CACHE, - 'success_responses', date_from, - date_to) + cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE, + 'success_responses', date_from, + date_to) current_use += cache_hits return current_use @@ -48,6 +48,17 @@ class UserService: if self._orgname: self.__increment_organization_uses(service_type, metric, date, amount) + def get_metrics(self, service, metric, date_from, date_to): + aggregated_metric = 0 + key_prefix = "org" if self._orgname else "user" + entity_name = self._orgname if self._orgname else self._username + for date in self.__generate_date_range(date_from, date_to): + redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, + service, metric, date) + score = self._redis_connection.zscore(redis_prefix, date.day) + aggregated_metric += score if score else 0 + return aggregated_metric + # Private functions def __increment_user_uses(self, service_type, metric, date, amount): @@ -68,17 +79,6 @@ class UserService: return redis_name - def __get_metrics(self, service, metric, date_from, date_to): - aggregated_metric = 0 - key_prefix = "org" if self._orgname else "user" - entity_name = self._orgname if self._orgname else self._username - for date in self.__generate_date_range(date_from, date_to): - redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, - service, metric, date) - score = self._redis_connection.zscore(redis_prefix, date.day) - aggregated_metric += score if score else 0 - return aggregated_metric - def __current_billing_cycle(self): """ Return the begining and end date for the current billing cycle """ end_period_day = self._user_geocoder_config.period_end_date.day diff --git a/server/lib/python/cartodb_geocoder/test/test_config_helper.py b/server/lib/python/cartodb_geocoder/test/test_config_helper.py index 061aab1..72b7d21 100644 --- a/server/lib/python/cartodb_geocoder/test/test_config_helper.py +++ b/server/lib/python/cartodb_geocoder/test/test_config_helper.py @@ -1,48 +1,41 @@ +import test_helper from cartodb_geocoder import config_helper from unittest import TestCase from nose.tools import assert_raises +from mockredis import MockRedis +from datetime import datetime, timedelta class TestConfigHelper(TestCase): - def test_should_return_list_of_user_config_if_its_ok(self): - user_config_json = '{"is_organization": false, "entity_name": "test_user"}' - user_config = config_helper.UserConfig(user_config_json, 'development_cartodb_user_UUID') - assert user_config.is_organization == False - assert user_config.entity_name == 'test_user' - assert user_config.user_id == 'UUID' + def setUp(self): + self.redis_conn = MockRedis() - def test_should_return_raise_config_exception_if_not_ok(self): - user_config_json = '{"is_organization": "false"}' - assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + 'test_user', None, + 'nokia_id', 'nokia_cod') + assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.geocoding_quota == 100 + assert geocoder_config.soft_geocoding_limit == False - def test_should_return_raise_config_exception_if_empty(self): - user_config_json = '{}' - assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): + yesterday = datetime.today() - timedelta(days=1) + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + test_helper.build_redis_org_config(self.redis_conn, 'test_org', + quota=200, end_date=yesterday) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + 'test_user', 'test_org', + 'nokia_id', 'nokia_cod') + assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.geocoding_quota == 200 + assert geocoder_config.soft_geocoding_limit == False + assert geocoder_config.period_end_date.date() == yesterday.date() - def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - geocoder_config_json = """{"street_geocoder_provider": "Nokia", - "nokia_monthly_quota": 100, "nokia_soft_geocoder_limit": false}""" - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) - assert geocoder_config.nokia_geocoder == True - assert geocoder_config.nokia_monthly_quota == 100 - assert geocoder_config.nokia_soft_limit == False - - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): - geocoder_config_json = '{"street_geocoder_provider": "NokiA", "nokia_monthly_quota": "100"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) - - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters_2(self): - geocoder_config_json = '{"street_geocoder_provider": "NoKia", "nokia_soft_geocoder_limit": "false"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) - - def test_should_return_list_of_google_geocoder_config_if_its_ok(self): - geocoder_config_json = """{"street_geocoder_provider": "gOOgle", - "google_maps_private_key": "sdasdasda"}""" - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) - assert geocoder_config.google_geocoder == True - assert geocoder_config.google_api_key == 'sdasdasda' - - def test_should_raise_configuration_exception_when_missing_google_api_key(self): - geocoder_config_json = '{"street_geocoder_provider": "google"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) \ No newline at end of file + def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + assert_raises(config_helper.ConfigException, + config_helper.GeocoderConfig, + self.redis_conn, 'test_user', + None, None, None) diff --git a/server/lib/python/cartodb_geocoder/test/test_helper.py b/server/lib/python/cartodb_geocoder/test/test_helper.py new file mode 100644 index 0000000..9f1f50f --- /dev/null +++ b/server/lib/python/cartodb_geocoder/test/test_helper.py @@ -0,0 +1,33 @@ +from datetime import datetime, date + + +def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, + service="heremaps", + end_date=datetime.today()): + user_redis_name = "rails:users:{0}".format(username) + redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit) + redis_conn.hset(user_redis_name, 'geocoding_quota', quota) + redis_conn.hset(user_redis_name, 'geocoder_type', service) + redis_conn.hset(user_redis_name, 'period_end_date', end_date) + redis_conn.hset(user_redis_name, 'google_maps_client_id', '') + redis_conn.hset(user_redis_name, 'google_maps_api_key', '') + + +def build_redis_org_config(redis_conn, orgname, quota=100, + end_date=datetime.today()): + org_redis_name = "rails:orgs:{0}".format(orgname) + redis_conn.hset(org_redis_name, 'geocoding_quota', quota) + redis_conn.hset(org_redis_name, 'period_end_date', end_date) + redis_conn.hset(org_redis_name, 'google_maps_client_id', '') + redis_conn.hset(org_redis_name, 'google_maps_api_key', '') + + +def increment_geocoder_uses(redis_conn, username, orgname=None, + date=date.today(), service='geocoder_here', + metric='success_responses', amount=20): + prefix = 'org' if orgname else 'user' + entity_name = orgname if orgname else username + yearmonth = date.strftime('%Y%m') + redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, + service, metric, yearmonth) + redis_conn.zincrby(redis_name, date.day, amount) diff --git a/server/lib/python/cartodb_geocoder/test/test_quota_service.py b/server/lib/python/cartodb_geocoder/test/test_quota_service.py index a618c67..7c71315 100644 --- a/server/lib/python/cartodb_geocoder/test/test_quota_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_quota_service.py @@ -1,85 +1,109 @@ +import test_helper from mockredis import MockRedis from cartodb_geocoder import quota_service from cartodb_geocoder import config_helper from unittest import TestCase from nose.tools import assert_raises -from datetime import datetime +from datetime import datetime, date class TestQuotaService(TestCase): - # single user - # user:::used_quota_month:year_month - # user:::used_quota_day:year_month_day - # organization user - # org:::used_quota_month:year_month - # org::::used_quota_day:year_month_day + # single user + # user::::YYYYMM:DD + # organization user + # org::::YYYYMM:DD - def setUp(self): - self.fake_redis_connection = MockRedis() +# def increment_geocoder_uses(self, username, orgname=None, +# date=date.today(), service='geocoder_here', +# metric='success_responses', amount=20): - def test_should_return_true_if_user_quota_with_no_use(self): - qs = self.__build_quota_service() - assert qs.check_user_quota() == True + def setUp(self): + self.redis_conn = MockRedis() - def test_should_return_true_if_org_quota_with_no_use(self): - qs = self.__build_quota_service(organization=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_with_no_use(self): + qs = self.__build_quota_service('test_user') + assert qs.check_user_quota() is True - def test_should_return_true_if_user_quota_is_not_completely_used(self): - qs = self.__build_quota_service() - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == True + def test_should_return_true_if_org_quota_with_no_use(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + assert qs.check_user_quota() is True - def test_should_return_true_if_org_quota_is_not_completely_used(self): - qs = self.__build_quota_service(organization=True) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_is_not_completely_used(self): + qs = self.__build_quota_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user') + assert qs.check_user_quota() is True - def test_should_return_false_if_user_quota_is_surpassed(self): - qs = self.__build_quota_service(quota = 1, soft_limit=False) - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == False + def test_should_return_true_if_org_quota_is_not_completely_used(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org') + assert qs.check_user_quota() is True - def test_should_return_false_if_org_quota_is_surpassed(self): - qs = self.__build_quota_service(organization=True, quota=1) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == False + def test_should_return_false_if_user_quota_is_surpassed(self): + qs = self.__build_quota_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=300) + assert qs.check_user_quota() is False - def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service(quota=1, soft_limit=True) - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == True + def test_should_return_false_if_org_quota_is_surpassed(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', amount=400) + assert qs.check_user_quota() is False - def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service(organization=True, quota=1, soft_limit=True) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service('test_user', soft_limit=True) + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=300) + assert qs.check_user_quota() is True - def test_should_check_user_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service(quota=2, soft_limit=False) - qs.increment_geocoder_use() - assert qs.check_user_quota() == True + def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service('test_user', orgname='test_org', + soft_limit=True) + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', amount=400) + assert qs.check_user_quota() is True - def test_should_check_org_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service(organization=True, quota=2, soft_limit=False) - qs.increment_geocoder_use() - assert qs.check_user_quota() == True + def test_should_check_user_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service('test_user', quota=2) + qs.increment_success_geocoder_use() + assert qs.check_user_quota() == True + qs.increment_success_geocoder_use(amount=2) + assert qs.check_user_quota() == False + month = date.today().strftime('%Y%m') + name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) + total_requests = self.redis_conn.zscore(name, date.today().day) + assert total_requests == 3 - def __build_quota_service(self, quota=100, service='nokia', organization=False, soft_limit=False): - is_organization = 'true' if organization else 'false' - has_soft_limit = 'true' if soft_limit else 'false' - user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) - geocoder_config_json = """{{"street_geocoder_provider": "{0}","nokia_monthly_quota": {1}, - "nokia_soft_geocoder_limit": {2}}}""".format(service, quota, has_soft_limit) - user_config = config_helper.UserConfig(user_config_json, 'user_id') - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) + def test_should_check_org_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service('test_user', orgname='test_org', + quota=2) + qs.increment_success_geocoder_use() + assert qs.check_user_quota() == True + qs.increment_success_geocoder_use(amount=2) + assert qs.check_user_quota() == False + month = date.today().strftime('%Y%m') + org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month) + org_total_requests = self.redis_conn.zscore(org_name, date.today().day) + assert org_total_requests == 3 + user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) + user_total_requests = self.redis_conn.zscore(user_name, date.today().day) + assert user_total_requests == 3 - return quota_service.QuotaService(user_config, geocoder_config, redis_connection = self.fake_redis_connection) + def __build_quota_service(self, username, quota=100, service='heremaps', + orgname=None, soft_limit=False, + end_date = datetime.today()): + test_helper.build_redis_user_config(self.redis_conn, username, + quota = quota, service = service, + soft_limit = soft_limit, + end_date = end_date) + if orgname: + test_helper.build_redis_org_config(self.redis_conn, orgname, + quota=quota, end_date=end_date) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return quota_service.QuotaService(geocoder_config, + redis_connection = self.redis_conn) - def __increment_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): - prefix = 'org' if org else 'user' - date = datetime.strptime(date_string, "%Y%m%d") - redis_name = "{0}:{1}:{2}:used_quota_month".format(prefix, entity_name, service) - redis_key_month = "{0}_{1}".format(date.year, date.month) - self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/test/test_user_service.py b/server/lib/python/cartodb_geocoder/test/test_user_service.py index 2e3b969..4aeb1c0 100644 --- a/server/lib/python/cartodb_geocoder/test/test_user_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_user_service.py @@ -1,63 +1,90 @@ +import test_helper from mockredis import MockRedis from cartodb_geocoder import user_service from cartodb_geocoder import config_helper -from datetime import datetime +from datetime import datetime, date from unittest import TestCase from nose.tools import assert_raises +from datetime import timedelta class TestUserService(TestCase): - NOKIA_GEOCODER = 'nokia' + NOKIA_GEOCODER = 'geocoder_here' def setUp(self): - self.fake_redis_connection = MockRedis() + self.redis_conn = MockRedis() - def test_user_used_quota_for_a_month(self): - us = self.__build_user_service() - self.__increment_month_geocoder_uses('test_user', '20151111') - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 + def test_user_used_quota_for_a_day(self): + us = self.__build_user_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=400) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 - def test_org_used_quota_for_a_month(self): - us = self.__build_user_service(organization=True) - self.__increment_month_geocoder_uses('test_user', '20151111', org=True) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 + def test_org_used_quota_for_a_day(self): + us = self.__build_user_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', + amount=400) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_user_not_amount_in_used_quota_for_month_should_be_0(self): - us = self.__build_user_service() - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 + us = self.__build_user_service('test_user') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0 def test_org_not_amount_in_used_quota_for_month_should_be_0(self): - us = self.__build_user_service(organization=True) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 + us = self.__build_user_service('test_user', orgname='test_org') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0 - def test_should_increment_user_used_quota(self): - us = self.__build_user_service() - date = datetime.strptime("20151111", "%Y%m%d") - us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 + def test_should_increment_user_used_quota_for_one_date(self): + us = self.__build_user_service('test_user') + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 def test_should_increment_org_used_quota(self): - us = self.__build_user_service(organization=True) - date = datetime.strptime("20151111", "%Y%m%d") - us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 + us = self.__build_user_service('test_user', orgname='test_org') + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 - def test_exception_if_not_redis_config(self): - assert_raises(Exception, user_service.UserService, 'user_id') + def test_should_increment_user_used_quota_in_for_multiples_dates(self): + two_days_ago = datetime.today() - timedelta(days=2) + one_day_ago = datetime.today() - timedelta(days=1) + one_day_after = datetime.today() + timedelta(days=1) + us = self.__build_user_service('test_user', end_date=one_day_ago) + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses', + date=date.today()) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=one_day_ago) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=two_days_ago) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=one_day_after) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 - def __build_user_service(self, organization=False, service='nokia'): - is_organization = 'true' if organization else 'false' - user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) - user_config = config_helper.UserConfig(user_config_json, 'user_id') - - return user_service.UserService(user_config, service, redis_connection = self.fake_redis_connection) - - def __increment_month_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): - parent_tag = 'org' if org else 'user' - date = datetime.strptime(date_string, "%Y%m%d") - redis_name = "{0}:{1}:{2}:used_quota_month".format(parent_tag, entity_name, service) - redis_key_month = "{0}_{1}".format(date.year, date.month) - self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file + def __build_user_service(self, username, quota=100, service='heremaps', + orgname=None, soft_limit=False, + end_date=datetime.today()): + test_helper.build_redis_user_config(self.redis_conn, username, + quota=quota, service=service, + soft_limit=soft_limit, + end_date=end_date) + if orgname: + test_helper.build_redis_org_config(self.redis_conn, orgname, + quota=quota, end_date=end_date) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return user_service.UserService(geocoder_config, self.redis_conn) From 984519e7f1adc95764d5f07a614842fa024507ac Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 09:52:19 +0100 Subject: [PATCH 15/40] Change to have functions for all the geocoder types but only one function to call them --- .../extension/sql/0.1.0/20_geocode_street.sql | 30 +++++++++++++++++-- .../cartodb_geocoder/config_helper.py | 19 +++++------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 941743e..c95662d 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -1,13 +1,31 @@ --- Geocodes a street address given a searchtext and a state and/or country +-- We have change arguments so we need to drop the former function DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT); + +-- Geocodes a street address given a searchtext and a state and/or country CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from heremaps import heremapsgeocoder from cartodb_geocoder import quota_service - plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] # -- Check the quota @@ -35,3 +53,9 @@ RETURNS Geometry AS $$ plpy.error(error_msg) $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.error('Google geocoder is not available yet') + return None +$$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index def1b47..9c069fc 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -75,23 +75,18 @@ class GeocoderConfig: else: self._orgname = None self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) - self._google_maps_api_key = None - self._google_maps_client_id = None - self._geocoding_quota = 0 - self._soft_geocoding_limit = False - if self.GOOGLE_GEOCODER == self._geocoder_type: - self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] - self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] - elif self.NOKIA_GEOCODER == self._geocoder_type: - self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) - self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] - self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] - if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: self._soft_geocoding_limit = False + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] + self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] @property def service_type(self): From bd8ec718555577d62398c2235903af331a3d90c8 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 10:37:40 +0100 Subject: [PATCH 16/40] Create update extension files for client too --- client/Makefile | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/client/Makefile b/client/Makefile index 1e0696c..910e347 100644 --- a/client/Makefile +++ b/client/Makefile @@ -2,8 +2,20 @@ # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_geocoder_client EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +SED = sed -DATA = $(EXTENSION)--$(EXTVERSION).sql +UPGRADABLE = \ + 0.0.1 \ + $(END) + +UPGRADES = \ + $(shell echo $(UPGRADABLE) | \ + $(SED) 's/^/$(EXTENSION)--/' | \ + $(SED) 's/$$/--$(EXTVERSION).sql/' | \ + $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g') + +DATA = $(EXTENSION)--$(EXTVERSION).sql \ + $(UPGRADES) REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) @@ -30,6 +42,9 @@ $(DATA): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@ +$(EXTENSION)--%--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql + cp $< $@ + all: $(DATA) # Only meant for development time, do not use once a version is released From 4bffdb5de186555f04ddcb9345f7d575b54cb90f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 10:38:34 +0100 Subject: [PATCH 17/40] Add default values for the parameters in the functions --- interface.yaml | 7 +++++++ sql-template-renderer | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/interface.yaml b/interface.yaml index c789cd3..e64c504 100644 --- a/interface.yaml +++ b/interface.yaml @@ -51,3 +51,10 @@ params: - { name: ip_address, type: text} +- name: cdb_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} diff --git a/sql-template-renderer b/sql-template-renderer index de5f9a7..5dff053 100755 --- a/sql-template-renderer +++ b/sql-template-renderer @@ -40,7 +40,14 @@ class SqlTemplateRenderer end def params_with_type - @function_signature['params'].map { |p| "#{p['name']} #{p['type']}"}.join(', ') + parameters = @function_signature['params'].map do |p| + if not p['default'].nil? + "#{p['name']} #{p['type']} DEFAULT #{p['default']}" + else + "#{p['name']} #{p['type']}" + end + end + return parameters.join(', ') end end From 49941a78f38848d7329a1b31d484a0cf24df3ebe Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 10:38:50 +0100 Subject: [PATCH 18/40] Change client version to 0.1.0 --- client/.gitignore | 2 +- client/cdb_geocoder_client.control | 2 +- client/sql/0.1.0/20_public_functions.sql | 260 ++++++++++++++++++++++ client/sql/0.1.0/30_plproxy_functions.sql | 60 +++++ client/sql/0.1.0/90_grant_execute.sql | 10 + 5 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 client/sql/0.1.0/20_public_functions.sql create mode 100644 client/sql/0.1.0/30_plproxy_functions.sql create mode 100644 client/sql/0.1.0/90_grant_execute.sql diff --git a/client/.gitignore b/client/.gitignore index 1ff37f5..093944e 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -1,4 +1,4 @@ results/ regression.diffs regression.out -cdb_geocoder_client--0.0.1.sql +cdb_geocoder_client--* diff --git a/client/cdb_geocoder_client.control b/client/cdb_geocoder_client.control index c903511..4d1f047 100644 --- a/client/cdb_geocoder_client.control +++ b/client/cdb_geocoder_client.control @@ -1,6 +1,6 @@ # CartoDB geocoder client API extension comment = 'CartoDB geocoder client API extension' -default_version = '0.0.1' +default_version = '0.1.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_geocoder_client diff --git a/client/sql/0.1.0/20_public_functions.sql b/client/sql/0.1.0/20_public_functions.sql new file mode 100644 index 0000000..ff5af39 --- /dev/null +++ b/client/sql/0.1.0/20_public_functions.sql @@ -0,0 +1,260 @@ +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + diff --git a/client/sql/0.1.0/30_plproxy_functions.sql b/client/sql/0.1.0/30_plproxy_functions.sql new file mode 100644 index 0000000..d87b1b9 --- /dev/null +++ b/client/sql/0.1.0/30_plproxy_functions.sql @@ -0,0 +1,60 @@ +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_street_point (username, organization_name, searchtext, city, state_province, country); +$$ LANGUAGE plproxy; + diff --git a/client/sql/0.1.0/90_grant_execute.sql b/client/sql/0.1.0/90_grant_execute.sql new file mode 100644 index 0000000..2f9b244 --- /dev/null +++ b/client/sql/0.1.0/90_grant_execute.sql @@ -0,0 +1,10 @@ +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point(searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) TO publicuser; From 7cce491a13d52e459e592ce360c407f8cc4786fd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 10:55:07 +0100 Subject: [PATCH 19/40] Dont use the defaults in the grant script --- client/sql/0.1.0/90_grant_execute.sql | 2 +- client/templates/20_public_functions.erb | 2 +- client/templates/30_plproxy_functions.erb | 2 +- sql-template-renderer | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/sql/0.1.0/90_grant_execute.sql b/client/sql/0.1.0/90_grant_execute.sql index 2f9b244..01ce6a4 100644 --- a/client/sql/0.1.0/90_grant_execute.sql +++ b/client/sql/0.1.0/90_grant_execute.sql @@ -7,4 +7,4 @@ GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_ GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point(searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; diff --git a/client/templates/20_public_functions.erb b/client/templates/20_public_functions.erb index 6ee473d..d213986 100644 --- a/client/templates/20_public_functions.erb +++ b/client/templates/20_public_functions.erb @@ -4,7 +4,7 @@ -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type %>) +CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ DECLARE ret <%= return_type %>; diff --git a/client/templates/30_plproxy_functions.erb b/client/templates/30_plproxy_functions.erb index 71ebdc4..906d8a5 100644 --- a/client/templates/30_plproxy_functions.erb +++ b/client/templates/30_plproxy_functions.erb @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type %>) +CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ CONNECT <%= GEOCODER_CLIENT_SCHEMA %>._server_conn_str(); SELECT cdb_geocoder_server.<%= name %> (username, organization_name, <%= params %>); diff --git a/sql-template-renderer b/sql-template-renderer index 5dff053..8938db8 100755 --- a/sql-template-renderer +++ b/sql-template-renderer @@ -40,6 +40,10 @@ class SqlTemplateRenderer end def params_with_type + @function_signature['params'].map { |p| "#{p['name']} #{p['type']}" }.join(', ') + end + + def params_with_type_and_default parameters = @function_signature['params'].map do |p| if not p['default'].nil? "#{p['name']} #{p['type']} DEFAULT #{p['default']}" @@ -49,7 +53,6 @@ class SqlTemplateRenderer end return parameters.join(', ') end - end From c0cef3dc80c60460b9f955b7f2c9f10378dc98e4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 12:55:36 +0100 Subject: [PATCH 20/40] Tests and interfaces versioned --- client/.gitignore | 6 +- client/Makefile | 20 +- client/cdb_geocoder_client--0.0.1--0.1.0.sql | 27 ++ client/cdb_geocoder_client--0.1.0--0.0.1.sql | 2 + client/sql/0.0.1/.gitignore | 3 - client/sql/0.1.0/00_header.sql | 2 + client/sql/0.1.0/10_geocoder_server_conn.sql | 16 ++ client/sql/0.1.0/15_config_management.sql | 37 +++ client/sql/0.1.0/20_public_functions.sql | 260 ------------------ client/sql/0.1.0/30_plproxy_functions.sql | 60 ---- client/sql/0.1.0/80_permissions.sql | 9 + client/sql/0.1.0/90_grant_execute.sql | 10 - .../0.0.1}/expected/00_installation_test.out | 0 .../0.0.1}/expected/10_admin0_test.out | 0 .../0.0.1}/expected/20_admin1_test.out | 0 .../0.0.1}/expected/30_namedplaces_test.out | 0 .../0.0.1}/expected/40_postalcodes_test.out | 0 .../0.0.1}/expected/50_ipaddresses_test.out | 0 .../0.0.1}/expected/90_permissions_test.out | 0 .../0.0.1}/sql/00_installation_test.sql | 0 .../{ => test/0.0.1}/sql/10_admin0_test.sql | 0 .../{ => test/0.0.1}/sql/20_admin1_test.sql | 0 .../0.0.1}/sql/30_namedplaces_test.sql | 0 .../0.0.1}/sql/40_postalcodes_test.sql | 0 .../0.0.1}/sql/50_ipaddresses_test.sql | 0 .../0.0.1}/sql/90_permissions_test.sql | 0 .../0.1.0/expected/00_installation_test.out | 29 ++ client/test/0.1.0/expected/10_admin0_test.out | 20 ++ client/test/0.1.0/expected/20_admin1_test.out | 36 +++ .../0.1.0/expected/30_namedplaces_test.out | 52 ++++ .../0.1.0/expected/40_postalcodes_test.out | 36 +++ .../0.1.0/expected/50_ipaddresses_test.out | 20 ++ .../test/0.1.0/expected/60_street_v2_test.out | 101 +++++++ .../0.1.0/expected/90_permissions_test.out | 117 ++++++++ .../test/0.1.0/sql/00_installation_test.sql | 23 ++ client/test/0.1.0/sql/10_admin0_test.sql | 15 + client/test/0.1.0/sql/20_admin1_test.sql | 24 ++ client/test/0.1.0/sql/30_namedplaces_test.sql | 33 +++ client/test/0.1.0/sql/40_postalcodes_test.sql | 23 ++ client/test/0.1.0/sql/50_ipaddresses_test.sql | 15 + client/test/0.1.0/sql/60_street_v2_test.sql | 24 ++ client/test/0.1.0/sql/90_permissions_test.sql | 30 ++ interface_0.0.1.yaml | 52 ++++ interface.yaml => interface_0.1.0.yaml | 2 +- 44 files changed, 754 insertions(+), 350 deletions(-) create mode 100644 client/cdb_geocoder_client--0.0.1--0.1.0.sql create mode 100644 client/cdb_geocoder_client--0.1.0--0.0.1.sql delete mode 100644 client/sql/0.0.1/.gitignore create mode 100644 client/sql/0.1.0/00_header.sql create mode 100644 client/sql/0.1.0/10_geocoder_server_conn.sql create mode 100644 client/sql/0.1.0/15_config_management.sql delete mode 100644 client/sql/0.1.0/20_public_functions.sql delete mode 100644 client/sql/0.1.0/30_plproxy_functions.sql create mode 100644 client/sql/0.1.0/80_permissions.sql delete mode 100644 client/sql/0.1.0/90_grant_execute.sql rename client/{ => test/0.0.1}/expected/00_installation_test.out (100%) rename client/{ => test/0.0.1}/expected/10_admin0_test.out (100%) rename client/{ => test/0.0.1}/expected/20_admin1_test.out (100%) rename client/{ => test/0.0.1}/expected/30_namedplaces_test.out (100%) rename client/{ => test/0.0.1}/expected/40_postalcodes_test.out (100%) rename client/{ => test/0.0.1}/expected/50_ipaddresses_test.out (100%) rename client/{ => test/0.0.1}/expected/90_permissions_test.out (100%) rename client/{ => test/0.0.1}/sql/00_installation_test.sql (100%) rename client/{ => test/0.0.1}/sql/10_admin0_test.sql (100%) rename client/{ => test/0.0.1}/sql/20_admin1_test.sql (100%) rename client/{ => test/0.0.1}/sql/30_namedplaces_test.sql (100%) rename client/{ => test/0.0.1}/sql/40_postalcodes_test.sql (100%) rename client/{ => test/0.0.1}/sql/50_ipaddresses_test.sql (100%) rename client/{ => test/0.0.1}/sql/90_permissions_test.sql (100%) create mode 100644 client/test/0.1.0/expected/00_installation_test.out create mode 100644 client/test/0.1.0/expected/10_admin0_test.out create mode 100644 client/test/0.1.0/expected/20_admin1_test.out create mode 100644 client/test/0.1.0/expected/30_namedplaces_test.out create mode 100644 client/test/0.1.0/expected/40_postalcodes_test.out create mode 100644 client/test/0.1.0/expected/50_ipaddresses_test.out create mode 100644 client/test/0.1.0/expected/60_street_v2_test.out create mode 100644 client/test/0.1.0/expected/90_permissions_test.out create mode 100644 client/test/0.1.0/sql/00_installation_test.sql create mode 100644 client/test/0.1.0/sql/10_admin0_test.sql create mode 100644 client/test/0.1.0/sql/20_admin1_test.sql create mode 100644 client/test/0.1.0/sql/30_namedplaces_test.sql create mode 100644 client/test/0.1.0/sql/40_postalcodes_test.sql create mode 100644 client/test/0.1.0/sql/50_ipaddresses_test.sql create mode 100644 client/test/0.1.0/sql/60_street_v2_test.sql create mode 100644 client/test/0.1.0/sql/90_permissions_test.sql create mode 100644 interface_0.0.1.yaml rename interface.yaml => interface_0.1.0.yaml (97%) diff --git a/client/.gitignore b/client/.gitignore index 093944e..cf79682 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -1,4 +1,8 @@ results/ regression.diffs regression.out -cdb_geocoder_client--* +20_public_functions.sql +30_plproxy_functions.sql +90_grant_execute.sql +cdb_geocoder_client--0.0.1.sql +cdb_geocoder_client--0.1.0.sql diff --git a/client/Makefile b/client/Makefile index 910e347..ee0cc6a 100644 --- a/client/Makefile +++ b/client/Makefile @@ -2,22 +2,12 @@ # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_geocoder_client EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -SED = sed -UPGRADABLE = \ - 0.0.1 \ - $(END) +DATA = $(EXTENSION)--$(EXTVERSION).sql -UPGRADES = \ - $(shell echo $(UPGRADABLE) | \ - $(SED) 's/^/$(EXTENSION)--/' | \ - $(SED) 's/$$/--$(EXTVERSION).sql/' | \ - $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g') - -DATA = $(EXTENSION)--$(EXTVERSION).sql \ - $(UPGRADES) - -REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) +REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) +TEST_DIR = test/$(EXTVERSION) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config @@ -28,7 +18,7 @@ SOURCES_DATA_DIR = sql/$(EXTVERSION) # The interface definition is used along with some templates to automatically generate code RENDERER = ../sql-template-renderer -INTERFACE_FILE = ../interface.yaml +INTERFACE_FILE = ../interface_$(EXTVERSION).yaml TEMPLATE_DIR = templates TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb) GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES)) diff --git a/client/cdb_geocoder_client--0.0.1--0.1.0.sql b/client/cdb_geocoder_client--0.0.1--0.1.0.sql new file mode 100644 index 0000000..a4503cf --- /dev/null +++ b/client/cdb_geocoder_client--0.0.1--0.1.0.sql @@ -0,0 +1,27 @@ +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2 (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point_v2 (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_street_point_v2 (username, organization_name, searchtext, city, state_province, country); +$$ LANGUAGE plproxy; + +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2(searchtext text, city text, state_province text, country text) TO publicuser; \ No newline at end of file diff --git a/client/cdb_geocoder_client--0.1.0--0.0.1.sql b/client/cdb_geocoder_client--0.1.0--0.0.1.sql new file mode 100644 index 0000000..b634d2f --- /dev/null +++ b/client/cdb_geocoder_client--0.1.0--0.0.1.sql @@ -0,0 +1,2 @@ +DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text) +DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text) \ No newline at end of file diff --git a/client/sql/0.0.1/.gitignore b/client/sql/0.0.1/.gitignore deleted file mode 100644 index 4cf1e95..0000000 --- a/client/sql/0.0.1/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -20_public_functions.sql -30_plproxy_functions.sql -90_grant_execute.sql diff --git a/client/sql/0.1.0/00_header.sql b/client/sql/0.1.0/00_header.sql new file mode 100644 index 0000000..a140c33 --- /dev/null +++ b/client/sql/0.1.0/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit diff --git a/client/sql/0.1.0/10_geocoder_server_conn.sql b/client/sql/0.1.0/10_geocoder_server_conn.sql new file mode 100644 index 0000000..5f3e1fc --- /dev/null +++ b/client/sql/0.1.0/10_geocoder_server_conn.sql @@ -0,0 +1,16 @@ +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/client/sql/0.1.0/15_config_management.sql b/client/sql/0.1.0/15_config_management.sql new file mode 100644 index 0000000..6ff3c8d --- /dev/null +++ b/client/sql/0.1.0/15_config_management.sql @@ -0,0 +1,37 @@ +CREATE TYPE cdb_geocoder_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; \ No newline at end of file diff --git a/client/sql/0.1.0/20_public_functions.sql b/client/sql/0.1.0/20_public_functions.sql deleted file mode 100644 index ff5af39..0000000 --- a/client/sql/0.1.0/20_public_functions.sql +++ /dev/null @@ -1,260 +0,0 @@ --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon (country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point (postal_code text, country_name text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point (ip_address text) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - --- --- Public geocoder API function --- --- These are the only ones with permissions to publicuser role --- and should also be the only ones with SECURITY DEFINER - -CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) -RETURNS Geometry AS $$ -DECLARE - ret Geometry; - username text; - orgname text; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument, check it out'; - END IF; - SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; - RETURN ret; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - diff --git a/client/sql/0.1.0/30_plproxy_functions.sql b/client/sql/0.1.0/30_plproxy_functions.sql deleted file mode 100644 index d87b1b9..0000000 --- a/client/sql/0.1.0/30_plproxy_functions.sql +++ /dev/null @@ -1,60 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) -RETURNS Geometry AS $$ - CONNECT cdb_geocoder_client._server_conn_str(); - SELECT cdb_geocoder_server.cdb_geocode_street_point (username, organization_name, searchtext, city, state_province, country); -$$ LANGUAGE plproxy; - diff --git a/client/sql/0.1.0/80_permissions.sql b/client/sql/0.1.0/80_permissions.sql new file mode 100644 index 0000000..16fc021 --- /dev/null +++ b/client/sql/0.1.0/80_permissions.sql @@ -0,0 +1,9 @@ +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser; diff --git a/client/sql/0.1.0/90_grant_execute.sql b/client/sql/0.1.0/90_grant_execute.sql deleted file mode 100644 index 01ce6a4..0000000 --- a/client/sql/0.1.0/90_grant_execute.sql +++ /dev/null @@ -1,10 +0,0 @@ -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; diff --git a/client/expected/00_installation_test.out b/client/test/0.0.1/expected/00_installation_test.out similarity index 100% rename from client/expected/00_installation_test.out rename to client/test/0.0.1/expected/00_installation_test.out diff --git a/client/expected/10_admin0_test.out b/client/test/0.0.1/expected/10_admin0_test.out similarity index 100% rename from client/expected/10_admin0_test.out rename to client/test/0.0.1/expected/10_admin0_test.out diff --git a/client/expected/20_admin1_test.out b/client/test/0.0.1/expected/20_admin1_test.out similarity index 100% rename from client/expected/20_admin1_test.out rename to client/test/0.0.1/expected/20_admin1_test.out diff --git a/client/expected/30_namedplaces_test.out b/client/test/0.0.1/expected/30_namedplaces_test.out similarity index 100% rename from client/expected/30_namedplaces_test.out rename to client/test/0.0.1/expected/30_namedplaces_test.out diff --git a/client/expected/40_postalcodes_test.out b/client/test/0.0.1/expected/40_postalcodes_test.out similarity index 100% rename from client/expected/40_postalcodes_test.out rename to client/test/0.0.1/expected/40_postalcodes_test.out diff --git a/client/expected/50_ipaddresses_test.out b/client/test/0.0.1/expected/50_ipaddresses_test.out similarity index 100% rename from client/expected/50_ipaddresses_test.out rename to client/test/0.0.1/expected/50_ipaddresses_test.out diff --git a/client/expected/90_permissions_test.out b/client/test/0.0.1/expected/90_permissions_test.out similarity index 100% rename from client/expected/90_permissions_test.out rename to client/test/0.0.1/expected/90_permissions_test.out diff --git a/client/sql/00_installation_test.sql b/client/test/0.0.1/sql/00_installation_test.sql similarity index 100% rename from client/sql/00_installation_test.sql rename to client/test/0.0.1/sql/00_installation_test.sql diff --git a/client/sql/10_admin0_test.sql b/client/test/0.0.1/sql/10_admin0_test.sql similarity index 100% rename from client/sql/10_admin0_test.sql rename to client/test/0.0.1/sql/10_admin0_test.sql diff --git a/client/sql/20_admin1_test.sql b/client/test/0.0.1/sql/20_admin1_test.sql similarity index 100% rename from client/sql/20_admin1_test.sql rename to client/test/0.0.1/sql/20_admin1_test.sql diff --git a/client/sql/30_namedplaces_test.sql b/client/test/0.0.1/sql/30_namedplaces_test.sql similarity index 100% rename from client/sql/30_namedplaces_test.sql rename to client/test/0.0.1/sql/30_namedplaces_test.sql diff --git a/client/sql/40_postalcodes_test.sql b/client/test/0.0.1/sql/40_postalcodes_test.sql similarity index 100% rename from client/sql/40_postalcodes_test.sql rename to client/test/0.0.1/sql/40_postalcodes_test.sql diff --git a/client/sql/50_ipaddresses_test.sql b/client/test/0.0.1/sql/50_ipaddresses_test.sql similarity index 100% rename from client/sql/50_ipaddresses_test.sql rename to client/test/0.0.1/sql/50_ipaddresses_test.sql diff --git a/client/sql/90_permissions_test.sql b/client/test/0.0.1/sql/90_permissions_test.sql similarity index 100% rename from client/sql/90_permissions_test.sql rename to client/test/0.0.1/sql/90_permissions_test.sql diff --git a/client/test/0.1.0/expected/00_installation_test.out b/client/test/0.1.0/expected/00_installation_test.out new file mode 100644 index 0000000..cbeef05 --- /dev/null +++ b/client/test/0.1.0/expected/00_installation_test.out @@ -0,0 +1,29 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; +-- Install the extension +CREATE EXTENSION cdb_geocoder_client; +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the server schema +CREATE SCHEMA cdb_geocoder_server; +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client; diff --git a/client/test/0.1.0/expected/10_admin0_test.out b/client/test/0.1.0/expected/10_admin0_test.out new file mode 100644 index 0000000..63dfbe0 --- /dev/null +++ b/client/test/0.1.0/expected/10_admin0_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params ("test_user", , Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/20_admin1_test.out b/client/test/0.1.0/expected/20_admin1_test.out new file mode 100644 index 0000000..3d5d055 --- /dev/null +++ b/client/test/0.1.0/expected/20_admin1_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California, United States) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/30_namedplaces_test.out b/client/test/0.1.0/expected/30_namedplaces_test.out new file mode 100644 index 0000000..3ca1a77 --- /dev/null +++ b/client/test/0.1.0/expected/30_namedplaces_test.out @@ -0,0 +1,52 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + diff --git a/client/test/0.1.0/expected/40_postalcodes_test.out b/client/test/0.1.0/expected/40_postalcodes_test.out new file mode 100644 index 0000000..1415fdd --- /dev/null +++ b/client/test/0.1.0/expected/40_postalcodes_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params ("test_user", , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params ("test_user", , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + diff --git a/client/test/0.1.0/expected/50_ipaddresses_test.out b/client/test/0.1.0/expected/50_ipaddresses_test.out new file mode 100644 index 0000000..624712d --- /dev/null +++ b/client/test/0.1.0/expected/50_ipaddresses_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params ("test_user", , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/60_street_v2_test.out b/client/test/0.1.0/expected/60_street_v2_test.out new file mode 100644 index 0000000..ad4e270 --- /dev/null +++ b/client/test/0.1.0/expected/60_street_v2_test.out @@ -0,0 +1,101 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point_v2('One street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/90_permissions_test.out b/client/test/0.1.0/expected/90_permissions_test.out new file mode 100644 index 0000000..351d0f9 --- /dev/null +++ b/client/test/0.1.0/expected/90_permissions_test.out @@ -0,0 +1,117 @@ +-- Use regular user role +SET ROLE test_regular_user; +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params ("test_user", , Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California, United States) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params ("test_user", , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params ("test_user", , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params ("test_user", , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('one street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , one street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin0_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_polygon +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_point +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +ERROR: permission denied for function _cdb_geocode_ipaddress_point +SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1'); +ERROR: permission denied for function _cdb_geocode_street_point_v2 diff --git a/client/test/0.1.0/sql/00_installation_test.sql b/client/test/0.1.0/sql/00_installation_test.sql new file mode 100644 index 0000000..784fa03 --- /dev/null +++ b/client/test/0.1.0/sql/00_installation_test.sql @@ -0,0 +1,23 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; + +-- Install the extension +CREATE EXTENSION cdb_geocoder_client; + +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + +-- Mock the server schema +CREATE SCHEMA cdb_geocoder_server; + +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client; \ No newline at end of file diff --git a/client/test/0.1.0/sql/10_admin0_test.sql b/client/test/0.1.0/sql/10_admin0_test.sql new file mode 100644 index 0000000..5c3e488 --- /dev/null +++ b/client/test/0.1.0/sql/10_admin0_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); diff --git a/client/test/0.1.0/sql/20_admin1_test.sql b/client/test/0.1.0/sql/20_admin1_test.sql new file mode 100644 index 0000000..a8a2075 --- /dev/null +++ b/client/test/0.1.0/sql/20_admin1_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); diff --git a/client/test/0.1.0/sql/30_namedplaces_test.sql b/client/test/0.1.0/sql/30_namedplaces_test.sql new file mode 100644 index 0000000..3309cf5 --- /dev/null +++ b/client/test/0.1.0/sql/30_namedplaces_test.sql @@ -0,0 +1,33 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); + diff --git a/client/test/0.1.0/sql/40_postalcodes_test.sql b/client/test/0.1.0/sql/40_postalcodes_test.sql new file mode 100644 index 0000000..b5aafc6 --- /dev/null +++ b/client/test/0.1.0/sql/40_postalcodes_test.sql @@ -0,0 +1,23 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); diff --git a/client/test/0.1.0/sql/50_ipaddresses_test.sql b/client/test/0.1.0/sql/50_ipaddresses_test.sql new file mode 100644 index 0000000..09a3342 --- /dev/null +++ b/client/test/0.1.0/sql/50_ipaddresses_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); diff --git a/client/test/0.1.0/sql/60_street_v2_test.sql b/client/test/0.1.0/sql/60_street_v2_test.sql new file mode 100644 index 0000000..99078c9 --- /dev/null +++ b/client/test/0.1.0/sql/60_street_v2_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point_v2('One street, 1'); +SELECT cdb_geocode_street_point_v2('One street', 'city'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +SELECT cdb_geocode_street_point_v2('One street, 1'); +SELECT cdb_geocode_street_point_v2('One street', 'city'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); \ No newline at end of file diff --git a/client/test/0.1.0/sql/90_permissions_test.sql b/client/test/0.1.0/sql/90_permissions_test.sql new file mode 100644 index 0000000..675867f --- /dev/null +++ b/client/test/0.1.0/sql/90_permissions_test.sql @@ -0,0 +1,30 @@ +-- Use regular user role +SET ROLE test_regular_user; + +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point_v2('one street, 1'); + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1'); diff --git a/interface_0.0.1.yaml b/interface_0.0.1.yaml new file mode 100644 index 0000000..9f219fb --- /dev/null +++ b/interface_0.0.1.yaml @@ -0,0 +1,52 @@ +--- +- name: cdb_geocode_admin0_polygon + return_type: Geometry + params: + - { name: country_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + - { name: country_name, type: text } + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: admin1_name, type: text} + - { name: country_name, type: text} + + +- name: cdb_geocode_postalcode_polygon + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_postalcode_point + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_ipaddress_point + return_type: Geometry + params: + - { name: ip_address, type: text} \ No newline at end of file diff --git a/interface.yaml b/interface_0.1.0.yaml similarity index 97% rename from interface.yaml rename to interface_0.1.0.yaml index e64c504..d5e6fed 100644 --- a/interface.yaml +++ b/interface_0.1.0.yaml @@ -51,7 +51,7 @@ params: - { name: ip_address, type: text} -- name: cdb_geocode_street_point +- name: cdb_geocode_street_point_v2 return_type: Geometry params: - { name: searchtext, type: text} From 56a4c60c8f29725b2fbe70661ad906a126e88942 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 15:32:16 +0100 Subject: [PATCH 21/40] Add already released cdb_geocoder_client--0.0.1.sql to VC --- client/cdb_geocoder_client--0.0.1.sql | 359 ++++++++++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 client/cdb_geocoder_client--0.0.1.sql diff --git a/client/cdb_geocoder_client--0.0.1.sql b/client/cdb_geocoder_client--0.0.1.sql new file mode 100644 index 0000000..5bcc3c8 --- /dev/null +++ b/client/cdb_geocoder_client--0.0.1.sql @@ -0,0 +1,359 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql';CREATE TYPE cdb_geocoder_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER;-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); +$$ LANGUAGE plproxy; + +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; From db3eaf31ad724009489fbdd3c358666fbc5e9a04 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 15:36:16 +0100 Subject: [PATCH 22/40] Versioned the server tests and created the upgrade and downgrade files --- server/extension/.gitignore | 4 +- server/extension/Makefile | 18 +- .../cdb_geocoder_server--0.0.1--0.1.0.sql | 132 ++++++++++++++ .../cdb_geocoder_server--0.1.0--0.0.1.sql | 6 + server/extension/sql/0.1.0/00_header.sql | 2 + .../extension/sql/0.1.0/10_redis_helper.sql | 15 +- .../extension/sql/0.1.0/20_geocode_street.sql | 5 +- server/extension/sql/0.1.0/30_admin0.sql | 37 ++++ server/extension/sql/0.1.0/40_admin1.sql | 89 ++++++++++ server/extension/sql/0.1.0/50_namedplaces.sql | 121 +++++++++++++ server/extension/sql/0.1.0/60_postalcodes.sql | 162 +++++++++++++++++ server/extension/sql/0.1.0/70_ips.sql | 49 ++++++ .../sql/0.1.0/90_geocoder_server_user.sql | 15 ++ .../0.0.1}/expected/00_install_test.out | 0 .../0.0.1}/expected/30_admin0_test.out | 0 .../0.0.1}/expected/40_admin1_test.out | 0 .../0.0.1}/expected/50_namedplaces_test.out | 0 .../0.0.1}/expected/60_postalcodes_test.out | 0 .../{ => test/0.0.1}/expected/70_ips_test.out | 0 .../90_remove_geocoder_api_user_test.out | 0 .../{ => test/0.0.1}/sql/00_install_test.sql | 0 .../{ => test/0.0.1}/sql/30_admin0_test.sql | 0 .../{ => test/0.0.1}/sql/40_admin1_test.sql | 0 .../0.0.1}/sql/50_namedplaces_test.sql | 0 .../0.0.1}/sql/60_postalcodes_test.sql | 0 .../{ => test/0.0.1}/sql/70_ips_test.sql | 0 .../sql/90_remove_geocoder_api_user_test.sql | 0 .../test/0.1.0/expected/00_install_test.out | 30 ++++ .../test/0.1.0/expected/30_admin0_test.out | 47 +++++ .../test/0.1.0/expected/40_admin1_test.out | 81 +++++++++ .../0.1.0/expected/50_namedplaces_test.out | 136 +++++++++++++++ .../0.1.0/expected/60_postalcodes_test.out | 163 ++++++++++++++++++ .../test/0.1.0/expected/70_ips_test.out | 40 +++++ .../90_remove_geocoder_api_user_test.out | 5 + .../test/0.1.0/sql/00_install_test.sql | 24 +++ .../test/0.1.0/sql/30_admin0_test.sql | 32 ++++ .../test/0.1.0/sql/40_admin1_test.sql | 48 ++++++ .../test/0.1.0/sql/50_namedplaces_test.sql | 72 ++++++++ .../test/0.1.0/sql/60_postalcodes_test.sql | 117 +++++++++++++ .../extension/test/0.1.0/sql/70_ips_test.sql | 24 +++ .../sql/90_remove_geocoder_api_user_test.sql | 5 + 41 files changed, 1453 insertions(+), 26 deletions(-) create mode 100644 server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql create mode 100644 server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql create mode 100644 server/extension/sql/0.1.0/00_header.sql create mode 100644 server/extension/sql/0.1.0/30_admin0.sql create mode 100644 server/extension/sql/0.1.0/40_admin1.sql create mode 100644 server/extension/sql/0.1.0/50_namedplaces.sql create mode 100644 server/extension/sql/0.1.0/60_postalcodes.sql create mode 100644 server/extension/sql/0.1.0/70_ips.sql create mode 100644 server/extension/sql/0.1.0/90_geocoder_server_user.sql rename server/extension/{ => test/0.0.1}/expected/00_install_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/30_admin0_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/40_admin1_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/50_namedplaces_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/60_postalcodes_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/70_ips_test.out (100%) rename server/extension/{ => test/0.0.1}/expected/90_remove_geocoder_api_user_test.out (100%) rename server/extension/{ => test/0.0.1}/sql/00_install_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/30_admin0_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/40_admin1_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/50_namedplaces_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/60_postalcodes_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/70_ips_test.sql (100%) rename server/extension/{ => test/0.0.1}/sql/90_remove_geocoder_api_user_test.sql (100%) create mode 100644 server/extension/test/0.1.0/expected/00_install_test.out create mode 100644 server/extension/test/0.1.0/expected/30_admin0_test.out create mode 100644 server/extension/test/0.1.0/expected/40_admin1_test.out create mode 100644 server/extension/test/0.1.0/expected/50_namedplaces_test.out create mode 100644 server/extension/test/0.1.0/expected/60_postalcodes_test.out create mode 100644 server/extension/test/0.1.0/expected/70_ips_test.out create mode 100644 server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out create mode 100644 server/extension/test/0.1.0/sql/00_install_test.sql create mode 100644 server/extension/test/0.1.0/sql/30_admin0_test.sql create mode 100644 server/extension/test/0.1.0/sql/40_admin1_test.sql create mode 100644 server/extension/test/0.1.0/sql/50_namedplaces_test.sql create mode 100644 server/extension/test/0.1.0/sql/60_postalcodes_test.sql create mode 100644 server/extension/test/0.1.0/sql/70_ips_test.sql create mode 100644 server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql diff --git a/server/extension/.gitignore b/server/extension/.gitignore index 340ecff..626ea9f 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -2,6 +2,4 @@ results/ regression.diffs regression.out cdb_geocoder_server--0.0.1.sql -cdb_geocoder_server--0.1.0.sql -cdb_geocoder_server--0.0.1--0.1.0.sql - +cdb_geocoder_server--0.1.0.sql \ No newline at end of file diff --git a/server/extension/Makefile b/server/extension/Makefile index b711756..4fb4daa 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -2,22 +2,13 @@ # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_geocoder_server EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -SED = sed - -UPGRADABLE = \ - 0.0.1 \ - $(END) - -UPGRADES = \ - $(shell echo $(UPGRADABLE) | \ - $(SED) 's/^/$(EXTENSION)--/' | \ - $(SED) 's/$$/--$(EXTVERSION).sql/' | \ - $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g') DATA = $(EXTENSION)--$(EXTVERSION).sql \ $(UPGRADES) -REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) +REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) +TEST_DIR = test/$(EXTVERSION) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config @@ -30,9 +21,6 @@ $(DATA): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA) >> $@ -$(EXTENSION)--%--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql - cp $< $@ - all: $(DATA) # Only meant for development time, do not use once a version is released diff --git a/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql new file mode 100644 index 0000000..a481c7a --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql @@ -0,0 +1,132 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + except heremapsgeocoder.EmptyGeocoderResponse: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.error('Google geocoder is not available yet') + return None +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql b/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql new file mode 100644 index 0000000..f626887 --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql @@ -0,0 +1,6 @@ +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf_v2(text); +DROP FUNCTION IF EXISTS cdb_geocoder_server._connect_to_redis(text); +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_geocoder_config(text, text); +DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point_v2(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); \ No newline at end of file diff --git a/server/extension/sql/0.1.0/00_header.sql b/server/extension/sql/0.1.0/00_header.sql new file mode 100644 index 0000000..949e06e --- /dev/null +++ b/server/extension/sql/0.1.0/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit diff --git a/server/extension/sql/0.1.0/10_redis_helper.sql b/server/extension/sql/0.1.0/10_redis_helper.sql index 5d7a43e..ef5047c 100644 --- a/server/extension/sql/0.1.0/10_redis_helper.sql +++ b/server/extension/sql/0.1.0/10_redis_helper.sql @@ -1,6 +1,13 @@ +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + -- Get the Redis configuration from the _conf table -- -DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf(); -CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) RETURNS cdb_geocoder_server._redis_conf_params AS $$ conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) conf = plpy.execute(conf_query)[0]['conf'] @@ -28,10 +35,10 @@ RETURNS boolean AS $$ from cartodb_geocoder import redis_helper metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, c.sentinel_master_id, c.timeout, c.redis_db - from cdb_geocoder_server._get_redis_conf('redis_metadata_config') c;""")[0] + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, c.sentinel_master_id, c.timeout, c.redis_db - from cdb_geocoder_server._get_redis_conf('redis_metrics_config') c;""")[0] + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], metadata_config_params['sentinel_port'], metadata_config_params['sentinel_master_id'], diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index c95662d..ccc4ce9 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -1,8 +1,5 @@ --- We have change arguments so we need to drop the former function -DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT); - -- Geocodes a street address given a searchtext and a state and/or country -CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] diff --git a/server/extension/sql/0.1.0/30_admin0.sql b/server/extension/sql/0.1.0/30_admin0.sql new file mode 100644 index 0000000..96bbc43 --- /dev/null +++ b/server/extension/sql/0.1.0/30_admin0.sql @@ -0,0 +1,37 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/40_admin1.sql b/server/extension/sql/0.1.0/40_admin1.sql new file mode 100644 index 0000000..44a1953 --- /dev/null +++ b/server/extension/sql/0.1.0/40_admin1.sql @@ -0,0 +1,89 @@ +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.1.0/50_namedplaces.sql b/server/extension/sql/0.1.0/50_namedplaces.sql new file mode 100644 index 0000000..44069f2 --- /dev/null +++ b/server/extension/sql/0.1.0/50_namedplaces.sql @@ -0,0 +1,121 @@ +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.1.0/60_postalcodes.sql b/server/extension/sql/0.1.0/60_postalcodes.sql new file mode 100644 index 0000000..1a20379 --- /dev/null +++ b/server/extension/sql/0.1.0/60_postalcodes.sql @@ -0,0 +1,162 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/70_ips.sql b/server/extension/sql/0.1.0/70_ips.sql new file mode 100644 index 0000000..5480c2d --- /dev/null +++ b/server/extension/sql/0.1.0/70_ips.sql @@ -0,0 +1,49 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/90_geocoder_server_user.sql b/server/extension/sql/0.1.0/90_geocoder_server_user.sql new file mode 100644 index 0000000..3c2b354 --- /dev/null +++ b/server/extension/sql/0.1.0/90_geocoder_server_user.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/expected/00_install_test.out b/server/extension/test/0.0.1/expected/00_install_test.out similarity index 100% rename from server/extension/expected/00_install_test.out rename to server/extension/test/0.0.1/expected/00_install_test.out diff --git a/server/extension/expected/30_admin0_test.out b/server/extension/test/0.0.1/expected/30_admin0_test.out similarity index 100% rename from server/extension/expected/30_admin0_test.out rename to server/extension/test/0.0.1/expected/30_admin0_test.out diff --git a/server/extension/expected/40_admin1_test.out b/server/extension/test/0.0.1/expected/40_admin1_test.out similarity index 100% rename from server/extension/expected/40_admin1_test.out rename to server/extension/test/0.0.1/expected/40_admin1_test.out diff --git a/server/extension/expected/50_namedplaces_test.out b/server/extension/test/0.0.1/expected/50_namedplaces_test.out similarity index 100% rename from server/extension/expected/50_namedplaces_test.out rename to server/extension/test/0.0.1/expected/50_namedplaces_test.out diff --git a/server/extension/expected/60_postalcodes_test.out b/server/extension/test/0.0.1/expected/60_postalcodes_test.out similarity index 100% rename from server/extension/expected/60_postalcodes_test.out rename to server/extension/test/0.0.1/expected/60_postalcodes_test.out diff --git a/server/extension/expected/70_ips_test.out b/server/extension/test/0.0.1/expected/70_ips_test.out similarity index 100% rename from server/extension/expected/70_ips_test.out rename to server/extension/test/0.0.1/expected/70_ips_test.out diff --git a/server/extension/expected/90_remove_geocoder_api_user_test.out b/server/extension/test/0.0.1/expected/90_remove_geocoder_api_user_test.out similarity index 100% rename from server/extension/expected/90_remove_geocoder_api_user_test.out rename to server/extension/test/0.0.1/expected/90_remove_geocoder_api_user_test.out diff --git a/server/extension/sql/00_install_test.sql b/server/extension/test/0.0.1/sql/00_install_test.sql similarity index 100% rename from server/extension/sql/00_install_test.sql rename to server/extension/test/0.0.1/sql/00_install_test.sql diff --git a/server/extension/sql/30_admin0_test.sql b/server/extension/test/0.0.1/sql/30_admin0_test.sql similarity index 100% rename from server/extension/sql/30_admin0_test.sql rename to server/extension/test/0.0.1/sql/30_admin0_test.sql diff --git a/server/extension/sql/40_admin1_test.sql b/server/extension/test/0.0.1/sql/40_admin1_test.sql similarity index 100% rename from server/extension/sql/40_admin1_test.sql rename to server/extension/test/0.0.1/sql/40_admin1_test.sql diff --git a/server/extension/sql/50_namedplaces_test.sql b/server/extension/test/0.0.1/sql/50_namedplaces_test.sql similarity index 100% rename from server/extension/sql/50_namedplaces_test.sql rename to server/extension/test/0.0.1/sql/50_namedplaces_test.sql diff --git a/server/extension/sql/60_postalcodes_test.sql b/server/extension/test/0.0.1/sql/60_postalcodes_test.sql similarity index 100% rename from server/extension/sql/60_postalcodes_test.sql rename to server/extension/test/0.0.1/sql/60_postalcodes_test.sql diff --git a/server/extension/sql/70_ips_test.sql b/server/extension/test/0.0.1/sql/70_ips_test.sql similarity index 100% rename from server/extension/sql/70_ips_test.sql rename to server/extension/test/0.0.1/sql/70_ips_test.sql diff --git a/server/extension/sql/90_remove_geocoder_api_user_test.sql b/server/extension/test/0.0.1/sql/90_remove_geocoder_api_user_test.sql similarity index 100% rename from server/extension/sql/90_remove_geocoder_api_user_test.sql rename to server/extension/test/0.0.1/sql/90_remove_geocoder_api_user_test.sql diff --git a/server/extension/test/0.1.0/expected/00_install_test.out b/server/extension/test/0.1.0/expected/00_install_test.out new file mode 100644 index 0000000..b386bec --- /dev/null +++ b/server/extension/test/0.1.0/expected/00_install_test.out @@ -0,0 +1,30 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); + cdb_setuserquotainbytes +------------------------- + 0 +(1 row) + diff --git a/server/extension/test/0.1.0/expected/30_admin0_test.out b/server/extension/test/0.1.0/expected/30_admin0_test.out new file mode 100644 index 0000000..52850c4 --- /dev/null +++ b/server/extension/test/0.1.0/expected/30_admin0_test.out @@ -0,0 +1,47 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/40_admin1_test.out b/server/extension/test/0.1.0/expected/40_admin1_test.out new file mode 100644 index 0000000..9897f34 --- /dev/null +++ b/server/extension/test/0.1.0/expected/40_admin1_test.out @@ -0,0 +1,81 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/50_namedplaces_test.out b/server/extension/test/0.1.0/expected/50_namedplaces_test.out new file mode 100644 index 0000000..1ac8d73 --- /dev/null +++ b/server/extension/test/0.1.0/expected/50_namedplaces_test.out @@ -0,0 +1,136 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/60_postalcodes_test.out b/server/extension/test/0.1.0/expected/60_postalcodes_test.out new file mode 100644 index 0000000..01aa003 --- /dev/null +++ b/server/extension/test/0.1.0/expected/60_postalcodes_test.out @@ -0,0 +1,163 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/70_ips_test.out b/server/extension/test/0.1.0/expected/70_ips_test.out new file mode 100644 index 0000000..2386200 --- /dev/null +++ b/server/extension/test/0.1.0/expected/70_ips_test.out @@ -0,0 +1,40 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +---------------------------------------------------- + 0101000020E61000003333333333334440AE47E17A14AE0D40 +(1 row) + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out b/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out new file mode 100644 index 0000000..c53fcfc --- /dev/null +++ b/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/extension/test/0.1.0/sql/00_install_test.sql b/server/extension/test/0.1.0/sql/00_install_test.sql new file mode 100644 index 0000000..8feac25 --- /dev/null +++ b/server/extension/test/0.1.0/sql/00_install_test.sql @@ -0,0 +1,24 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.1.0/sql/30_admin0_test.sql b/server/extension/test/0.1.0/sql/30_admin0_test.sql new file mode 100644 index 0000000..3851d4f --- /dev/null +++ b/server/extension/test/0.1.0/sql/30_admin0_test.sql @@ -0,0 +1,32 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); + +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/40_admin1_test.sql b/server/extension/test/0.1.0/sql/40_admin1_test.sql new file mode 100644 index 0000000..d3080bf --- /dev/null +++ b/server/extension/test/0.1.0/sql/40_admin1_test.sql @@ -0,0 +1,48 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); + +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/50_namedplaces_test.sql b/server/extension/test/0.1.0/sql/50_namedplaces_test.sql new file mode 100644 index 0000000..47c304a --- /dev/null +++ b/server/extension/test/0.1.0/sql/50_namedplaces_test.sql @@ -0,0 +1,72 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); + +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); + +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/60_postalcodes_test.sql b/server/extension/test/0.1.0/sql/60_postalcodes_test.sql new file mode 100644 index 0000000..2a8192e --- /dev/null +++ b/server/extension/test/0.1.0/sql/60_postalcodes_test.sql @@ -0,0 +1,117 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/70_ips_test.sql b/server/extension/test/0.1.0/sql/70_ips_test.sql new file mode 100644 index 0000000..f9875f1 --- /dev/null +++ b/server/extension/test/0.1.0/sql/70_ips_test.sql @@ -0,0 +1,24 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql b/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql new file mode 100644 index 0000000..4efb88e --- /dev/null +++ b/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; \ No newline at end of file From ba7af1ca2c49d3422091335e243b41a3ae4c0693 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 15:50:54 +0100 Subject: [PATCH 23/40] Remove uneeded vars in Makefile --- client/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/Makefile b/client/Makefile index ee0cc6a..717c2e5 100644 --- a/client/Makefile +++ b/client/Makefile @@ -6,8 +6,6 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/defau DATA = $(EXTENSION)--$(EXTVERSION).sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) -TEST_DIR = test/$(EXTVERSION) -REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config From d9c016d2428fd20db1de3e91eb3dc1d35b85ba6c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 15:51:05 +0100 Subject: [PATCH 24/40] Remove uneeded target from Makefile --- client/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/Makefile b/client/Makefile index 717c2e5..fcd9a56 100644 --- a/client/Makefile +++ b/client/Makefile @@ -30,9 +30,6 @@ $(DATA): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@ -$(EXTENSION)--%--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql - cp $< $@ - all: $(DATA) # Only meant for development time, do not use once a version is released From 9062ff8b749c35d41954439b939c8b66587e6e31 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 15:56:17 +0100 Subject: [PATCH 25/40] Added server tests for the street functions --- .../extension/test/0.1.0/expected/20_street_test.out | 12 ++++++++++++ server/extension/test/0.1.0/sql/20_street_test.sql | 7 +++++++ 2 files changed, 19 insertions(+) create mode 100644 server/extension/test/0.1.0/expected/20_street_test.out create mode 100644 server/extension/test/0.1.0/sql/20_street_test.sql diff --git a/server/extension/test/0.1.0/expected/20_street_test.out b/server/extension/test/0.1.0/expected/20_street_test.out new file mode 100644 index 0000000..526cc00 --- /dev/null +++ b/server/extension/test/0.1.0/expected/20_street_test.out @@ -0,0 +1,12 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/sql/20_street_test.sql b/server/extension/test/0.1.0/sql/20_street_test.sql new file mode 100644 index 0000000..7613f5b --- /dev/null +++ b/server/extension/test/0.1.0/sql/20_street_test.sql @@ -0,0 +1,7 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); \ No newline at end of file From 97a918306fe741d53a90ac021f70b35ef0cb62f4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 16:14:11 +0100 Subject: [PATCH 26/40] Modify the Makefile to install everything in place Install everything by using the underlying postgres extension makefiles infrastructure. --- client/Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/client/Makefile b/client/Makefile index fcd9a56..6eb3548 100644 --- a/client/Makefile +++ b/client/Makefile @@ -3,7 +3,18 @@ EXTENSION = cdb_geocoder_client EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -DATA = $(EXTENSION)--$(EXTVERSION).sql +# The new version to be generated from templates +NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql + +# DATA is a special variable used by postgres build infrastructure +# These are the files to be installed in the server shared dir, +# for installation from scratch, upgrades and downgrades. +# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html +DATA = $(NEW_EXTENSION_ARTIFACT) \ + cdb_geocoder_client--0.0.1.sql \ + cdb_geocoder_client--0.1.0--0.0.1.sql \ + cdb_geocoder_client--0.0.1--0.1.0.sql + REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) @@ -26,7 +37,7 @@ $(GENERATED_SQL_FILES): $(SOURCES_DATA_DIR)/%.sql: $(TEMPLATE_DIR)/%.erb $(INTER SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES) -$(DATA): $(SOURCES_DATA) +$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@ @@ -34,5 +45,5 @@ all: $(DATA) # Only meant for development time, do not use once a version is released devclean: - rm -f $(DATA) + rm -f $(NEW_EXTENSION_ARTIFACT) rm -f $(GENERATED_SQL_FILES) From bd4a343e93ffe6175883c91d0232490cc569a2df Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 16:19:15 +0100 Subject: [PATCH 27/40] Fix typo (missing semicolons) --- client/cdb_geocoder_client--0.1.0--0.0.1.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/cdb_geocoder_client--0.1.0--0.0.1.sql b/client/cdb_geocoder_client--0.1.0--0.0.1.sql index b634d2f..e8b4837 100644 --- a/client/cdb_geocoder_client--0.1.0--0.0.1.sql +++ b/client/cdb_geocoder_client--0.1.0--0.0.1.sql @@ -1,2 +1,2 @@ -DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text) -DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text) \ No newline at end of file +DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text); +DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text); From c58d7229a2e6847efd586534fce22033e5e9c3dc Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 16:23:31 +0100 Subject: [PATCH 28/40] Changed the regress part of the makefile to execute the tests in other location --- client/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/Makefile b/client/Makefile index 6eb3548..a9ff0c0 100644 --- a/client/Makefile +++ b/client/Makefile @@ -17,6 +17,8 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) +TEST_DIR = test/$(EXTVERSION) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config From 54b45c80760d0da50efd6c55a5dceb12670f5fbd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 16:30:38 +0100 Subject: [PATCH 29/40] Add schema for the get config function --- client/cdb_geocoder_client--0.0.1--0.1.0.sql | 2 +- client/cdb_geocoder_client--0.0.1.sql | 18 +++++++++--------- client/templates/20_public_functions.erb | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/cdb_geocoder_client--0.0.1--0.1.0.sql b/client/cdb_geocoder_client--0.0.1--0.1.0.sql index a4503cf..d26014d 100644 --- a/client/cdb_geocoder_client--0.0.1--0.1.0.sql +++ b/client/cdb_geocoder_client--0.0.1--0.1.0.sql @@ -8,7 +8,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/cdb_geocoder_client--0.0.1.sql b/client/cdb_geocoder_client--0.0.1.sql index 5bcc3c8..846931c 100644 --- a/client/cdb_geocoder_client--0.0.1.sql +++ b/client/cdb_geocoder_client--0.0.1.sql @@ -67,7 +67,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -93,7 +93,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -119,7 +119,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -145,7 +145,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -171,7 +171,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -197,7 +197,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -223,7 +223,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -249,7 +249,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; @@ -275,7 +275,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/templates/20_public_functions.erb b/client/templates/20_public_functions.erb index d213986..39a7c06 100644 --- a/client/templates/20_public_functions.erb +++ b/client/templates/20_public_functions.erb @@ -14,7 +14,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM <%= GEOCODER_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; From e46600325b5df8ec473708131af7a9b0c02222bd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 16:39:55 +0100 Subject: [PATCH 30/40] Use the real heremaps url --- server/lib/python/heremaps/heremaps/heremapsgeocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index dd4bdb8..0e7852d 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -80,7 +80,7 @@ class Geocoder: 'gen': self.gen } request_params.update(params) - response = requests.get(self.URL_DEV_GEOCODE_JSON, params=request_params) + response = requests.get(self.URL_GEOCODE_JSON, params=request_params) if response.status_code == requests.codes.ok: return json.loads(response.text) else: From d3d6147b8ee652985b398efcc2726db78e12a13f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 17:24:18 +0100 Subject: [PATCH 31/40] Server extension makefile migrate the upgrades and downgrades too --- server/extension/Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/server/extension/Makefile b/server/extension/Makefile index 4fb4daa..daa1d05 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -3,8 +3,18 @@ EXTENSION = cdb_geocoder_server EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -DATA = $(EXTENSION)--$(EXTVERSION).sql \ - $(UPGRADES) +# The new version to be generated from templates +NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql + +# DATA is a special variable used by postgres build infrastructure +# These are the files to be installed in the server shared dir, +# for installation from scratch, upgrades and downgrades. +# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html +DATA = $(NEW_EXTENSION_ARTIFACT) \ + cdb_geocoder_server--0.0.1.sql \ + cdb_geocoder_server--0.1.0--0.0.1.sql \ + cdb_geocoder_server--0.0.1--0.1.0.sql + REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) TEST_DIR = test/$(EXTVERSION) @@ -21,6 +31,10 @@ $(DATA): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA) >> $@ +$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) + rm -f $@ + cat $(SOURCES_DATA_DIR)/*.sql >> $@ + all: $(DATA) # Only meant for development time, do not use once a version is released From c129a8c82d508d79a0b228645cc7551cd4d263b0 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 18:20:04 +0100 Subject: [PATCH 32/40] Fix makefile Populate missing variable and also make sure devclean deletes the right files. --- server/extension/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/extension/Makefile b/server/extension/Makefile index daa1d05..f42de4a 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -25,6 +25,8 @@ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +SOURCES_DATA_DIR = sql/$(EXTVERSION) + SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) $(DATA): $(SOURCES_DATA) @@ -39,4 +41,4 @@ all: $(DATA) # Only meant for development time, do not use once a version is released devclean: - rm -f $(DATA) + rm -f $(NEW_EXTENSION_ARTIFACT) From 63651f512b04588b376f9285da9e4f865c32c6bc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 22 Jan 2016 18:30:26 +0100 Subject: [PATCH 33/40] Commit cdb_geocoder_server--0.0.1.sql into VC After making sure it is the same version as in production. Same as generated from last 0.0.1beta2 tag. --- .../extension/cdb_geocoder_server--0.0.1.sql | 539 ++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 server/extension/cdb_geocoder_server--0.0.1.sql diff --git a/server/extension/cdb_geocoder_server--0.0.1.sql b/server/extension/cdb_geocoder_server--0.0.1.sql new file mode 100644 index 0000000..33325df --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.0.1.sql @@ -0,0 +1,539 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf() +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('redis_conf') conf")[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + if user_id in GD and 'redis_connection' in GD[user_id]: + return False + else: + from cartodb_geocoder import redis_helper + config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf() c;""")[0] + redis_connection = redis_helper.RedisHelper(config_params['sentinel_host'], + config_params['sentinel_port'], + config_params['sentinel_master_id'], + timeout=config_params['timeout'], + redis_db=config_params['redis_db']).redis_connection() + GD[user_id] = {'redis_connection': redis_connection} + return True +$$ LANGUAGE plpythonu;-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) + RETURNS Geometry +AS $$ + import json + from heremaps import heremapsgeocoder + + heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf']) + + app_id = heremaps_conf['geocoder']['app_id'] + app_code = heremaps_conf['geocoder']['app_code'] + + geocoder = heremapsgeocoder.Geocoder(app_id, app_code) + + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + + return point['st_setsrid'] +$$ LANGUAGE plpythonu;-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file From 1a8109ecdf75d5c6447e0a2b5ba50f63eea9542e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 19:17:16 +0100 Subject: [PATCH 34/40] Quote NULL values to avoid treat the NULL as a string --- server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql | 2 +- server/extension/sql/0.1.0/20_geocode_street.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql index a481c7a..43aa150 100644 --- a/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql +++ b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql @@ -77,7 +77,7 @@ CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(usern RETURNS Geometry AS $$ plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] if user_geocoder_config.heremaps_geocoder: diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index ccc4ce9..3eac170 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(usern RETURNS Geometry AS $$ plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config('{0}', '{1}')".format(username, orgname)) + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quota_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] if user_geocoder_config.heremaps_geocoder: From 62ade7f1a0cae7ddfa9d7ed287000c30de3c762e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 22 Jan 2016 19:18:34 +0100 Subject: [PATCH 35/40] Extract the username and the orgname as strings --- client/cdb_geocoder_client--0.0.1--0.1.0.sql | 33 +++++++++++++++++++ client/cdb_geocoder_client--0.1.0--0.0.1.sql | 33 +++++++++++++++++++ client/sql/0.1.0/15_config_management.sql | 4 +-- client/test/0.1.0/expected/10_admin0_test.out | 2 +- client/test/0.1.0/expected/20_admin1_test.out | 4 +-- .../0.1.0/expected/30_namedplaces_test.out | 6 ++-- .../0.1.0/expected/40_postalcodes_test.out | 4 +-- .../0.1.0/expected/50_ipaddresses_test.out | 2 +- .../test/0.1.0/expected/60_street_v2_test.out | 20 +++++------ .../0.1.0/expected/90_permissions_test.out | 20 +++++------ 10 files changed, 97 insertions(+), 31 deletions(-) diff --git a/client/cdb_geocoder_client--0.0.1--0.1.0.sql b/client/cdb_geocoder_client--0.0.1--0.1.0.sql index d26014d..76032ad 100644 --- a/client/cdb_geocoder_client--0.0.1--0.1.0.sql +++ b/client/cdb_geocoder_client--0.0.1--0.1.0.sql @@ -1,3 +1,36 @@ +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2 (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE diff --git a/client/cdb_geocoder_client--0.1.0--0.0.1.sql b/client/cdb_geocoder_client--0.1.0--0.0.1.sql index e8b4837..c10a677 100644 --- a/client/cdb_geocoder_client--0.1.0--0.0.1.sql +++ b/client/cdb_geocoder_client--0.1.0--0.0.1.sql @@ -1,2 +1,35 @@ DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text); DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; \ No newline at end of file diff --git a/client/sql/0.1.0/15_config_management.sql b/client/sql/0.1.0/15_config_management.sql index 6ff3c8d..34da860 100644 --- a/client/sql/0.1.0/15_config_management.sql +++ b/client/sql/0.1.0/15_config_management.sql @@ -25,9 +25,9 @@ BEGIN FROM pg_namespace s LEFT JOIN pg_roles r ON s.nspowner = r.oid WHERE r.rolname = session_user INTO username; - SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; ELSE - SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; organization_name = NULL; END IF; result.username = username; diff --git a/client/test/0.1.0/expected/10_admin0_test.out b/client/test/0.1.0/expected/10_admin0_test.out index 63dfbe0..46d0b3f 100644 --- a/client/test/0.1.0/expected/10_admin0_test.out +++ b/client/test/0.1.0/expected/10_admin0_test.out @@ -10,7 +10,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params ("test_user", , Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement cdb_geocode_admin0_polygon diff --git a/client/test/0.1.0/expected/20_admin1_test.out b/client/test/0.1.0/expected/20_admin1_test.out index 3d5d055..2b859c3 100644 --- a/client/test/0.1.0/expected/20_admin1_test.out +++ b/client/test/0.1.0/expected/20_admin1_test.out @@ -17,7 +17,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California) +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement cdb_geocode_admin1_polygon @@ -26,7 +26,7 @@ PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California, United States) +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement cdb_geocode_admin1_polygon diff --git a/client/test/0.1.0/expected/30_namedplaces_test.out b/client/test/0.1.0/expected/30_namedplaces_test.out index 3ca1a77..4205dc0 100644 --- a/client/test/0.1.0/expected/30_namedplaces_test.out +++ b/client/test/0.1.0/expected/30_namedplaces_test.out @@ -24,7 +24,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement cdb_geocode_namedplace_point @@ -33,7 +33,7 @@ PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement cdb_geocode_namedplace_point @@ -42,7 +42,7 @@ PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL stateme (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement cdb_geocode_namedplace_point diff --git a/client/test/0.1.0/expected/40_postalcodes_test.out b/client/test/0.1.0/expected/40_postalcodes_test.out index 1415fdd..92d39d1 100644 --- a/client/test/0.1.0/expected/40_postalcodes_test.out +++ b/client/test/0.1.0/expected/40_postalcodes_test.out @@ -17,7 +17,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params ("test_user", , 03204, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement cdb_geocode_postalcode_polygon @@ -26,7 +26,7 @@ PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL state (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params ("test_user", , 03204, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement cdb_geocode_postalcode_point diff --git a/client/test/0.1.0/expected/50_ipaddresses_test.out b/client/test/0.1.0/expected/50_ipaddresses_test.out index 624712d..9a10179 100644 --- a/client/test/0.1.0/expected/50_ipaddresses_test.out +++ b/client/test/0.1.0/expected/50_ipaddresses_test.out @@ -10,7 +10,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params ("test_user", , 8.8.8.8) +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement cdb_geocode_ipaddress_point diff --git a/client/test/0.1.0/expected/60_street_v2_test.out b/client/test/0.1.0/expected/60_street_v2_test.out index ad4e270..6750855 100644 --- a/client/test/0.1.0/expected/60_street_v2_test.out +++ b/client/test/0.1.0/expected/60_street_v2_test.out @@ -10,7 +10,7 @@ END; $$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT cdb_geocode_street_point_v2('One street, 1'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, 1, , , ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, 1, , , ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -19,7 +19,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -28,7 +28,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -37,7 +37,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, country) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, country) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -46,7 +46,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , country) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , country) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -55,7 +55,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street, 1'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, 1, , , ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, 1, , , ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -64,7 +64,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -73,7 +73,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -82,7 +82,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, state, country) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, country) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 @@ -91,7 +91,7 @@ PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQ (1 row) SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , One street, city, , country) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , country) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 diff --git a/client/test/0.1.0/expected/90_permissions_test.out b/client/test/0.1.0/expected/90_permissions_test.out index 351d0f9..7569760 100644 --- a/client/test/0.1.0/expected/90_permissions_test.out +++ b/client/test/0.1.0/expected/90_permissions_test.out @@ -5,7 +5,7 @@ SET search_path TO public,cartodb,cdb_geocoder_client; -- Exercise the public function -- it is public, it shall work SELECT cdb_geocode_admin0_polygon('Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params ("test_user", , Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement cdb_geocode_admin0_polygon @@ -14,7 +14,7 @@ PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_admin1_polygon('California'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California) +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement cdb_geocode_admin1_polygon @@ -23,7 +23,7 @@ PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_admin1_polygon('California', 'United States'); -NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params ("test_user", , California, United States) +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement cdb_geocode_admin1_polygon @@ -32,7 +32,7 @@ PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement (1 row) SELECT cdb_geocode_namedplace_point('Elx'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement cdb_geocode_namedplace_point @@ -41,7 +41,7 @@ PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement cdb_geocode_namedplace_point @@ -50,7 +50,7 @@ PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL stateme (1 row) SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params ("test_user", , Elx, Valencia, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement cdb_geocode_namedplace_point @@ -59,7 +59,7 @@ PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL st (1 row) SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params ("test_user", , 03204, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement cdb_geocode_postalcode_polygon @@ -68,7 +68,7 @@ PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL state (1 row) SELECT cdb_geocode_postalcode_point('03204', 'Spain'); -NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params ("test_user", , 03204, Spain) +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement cdb_geocode_postalcode_point @@ -77,7 +77,7 @@ PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL stateme (1 row) SELECT cdb_geocode_ipaddress_point('8.8.8.8'); -NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params ("test_user", , 8.8.8.8) +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement cdb_geocode_ipaddress_point @@ -86,7 +86,7 @@ PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement (1 row) SELECT cdb_geocode_street_point_v2('one street, 1'); -NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params ("test_user", , one street, 1, , , ) +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , one street, 1, , , ) CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement cdb_geocode_street_point_v2 From fffcef90a5722cd95b071fed33441ffa7be11ce2 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 25 Jan 2016 08:17:03 +0100 Subject: [PATCH 36/40] Check for the mandatories configurations in Redis before proceed --- .../cartodb_geocoder/config_helper.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 9c069fc..0c3878e 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -41,20 +41,29 @@ class GeocoderConfig: heremaps_app_code=None): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) - user_config[self.USERNAME_KEY] = username - user_config[self.ORGNAME_KEY] = orgname - user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id - user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code - if orgname: - org_config = self._redis_connection.hgetall( - "rails:orgs:{0}".format(orgname)) + if not user_config: + raise ConfigException("""There is no user config available. Please check your configuration.'""") + else: + user_config[self.USERNAME_KEY] = username + user_config[self.ORGNAME_KEY] = orgname + user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id + user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code + if orgname: + self.__get_organization_config(orgname, user_config) + + return user_config + + def __get_organization_config(self, orgname, user_config): + org_config = self._redis_connection.hgetall( + "rails:orgs:{0}".format(orgname)) + if not org_config: + raise ConfigException("""There is no organization config available. Please check your configuration.'""") + else: user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] - return user_config - def __check_config(self, filtered_config): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): @@ -63,8 +72,7 @@ class GeocoderConfig: raise ConfigException("""Nokia geocoder configuration is missing. Check it please""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): - raise ConfigException("""Google geocoder need the mandatory - parameter 'google_maps_private_key'""") + raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") return True From 326c5e0f7d5d86c753a8409cf3f8def33e05abf1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 25 Jan 2016 11:00:42 +0100 Subject: [PATCH 37/40] Added street geocoder functions to integration tests --- test/README.md | 1 + test/fixtures/geocoder_api_test_dataset.csv | 6 ++-- test/integration/test_street_functions.py | 32 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 test/integration/test_street_functions.py diff --git a/test/README.md b/test/README.md index d1e8028..1eb6229 100644 --- a/test/README.md +++ b/test/README.md @@ -18,3 +18,4 @@ This suite of tests test the following parts of the geocoding API through the SQ - Named places functions - Postal code functions - Ip address functions +- Street address functions (This will call Heremaps or Google so it will cost you 2 credits) diff --git a/test/fixtures/geocoder_api_test_dataset.csv b/test/fixtures/geocoder_api_test_dataset.csv index b27327b..072a95c 100644 --- a/test/fixtures/geocoder_api_test_dataset.csv +++ b/test/fixtures/geocoder_api_test_dataset.csv @@ -1,3 +1,3 @@ -id,country,province,city,postalcode,ip -1,Spain,Castilla y León,Valladolid,47010,8.8.8.8 -2,USA,New York,Manhattn,10001,8.8.8.8 \ No newline at end of file +id,country,province,city,postalcode,ip,street +1,Spain,Castilla y León,Valladolid,47010,8.8.8.8,Calle Amor de Dios +2,USA,New York,Manhattn,10001,8.8.8.8,NonExistentStreet diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py new file mode 100644 index 0000000..5af715c --- /dev/null +++ b/test/integration/test_street_functions.py @@ -0,0 +1,32 @@ +from unittest import TestCase +from nose.tools import assert_raises +from nose.tools import assert_not_equal, assert_equal +from ..helpers.integration_test_helper import IntegrationTestHelper + + +class TestStreetFunctions(TestCase): + + def setUp(self): + self.env_variables = IntegrationTestHelper.get_environment_variables() + self.sql_api_url = "https://{0}.{1}/api/v2/sql".format( + self.env_variables['username'], + self.env_variables['host'], + self.env_variables['api_key'] + ) + + def test_if_select_with_street_point_is_ok(self): + query = "SELECT cdb_geocode_street_point_v2_(street) " \ + "as geometry FROM {0} LIMIT 1&api_key={1}".format( + self.env_variables['table_name'], + self.env_variables['api_key']) + geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) + assert_not_equal(geometry, None) + + def test_if_select_with_street_without_api_key_raise_error(self): + query = "SELECT cdb_geocode_street_point_v2_(street) " \ + "as geometry FROM {0} LIMIT 1".format( + self.env_variables['table_name']) + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") From 90ba8c88d126fa8753635c45374d7a472e909b00 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 25 Jan 2016 16:22:00 +0100 Subject: [PATCH 38/40] Add deploy instructions to readme --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 0b12815..d80e222 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # geocoder-api The CartoDB Geocoder SQL API (server and client FTM) + +### Deploy instructions +Steps to deploy a new Geocoder API version : + +- Deploy new version of geocoder API to all servers +- Update the server user using: ALTER EXTENSION cdb_geocoder_server UPDATE TO ''; +- Update the python dependencies if needed: **cartodb_geocoder** and **heremaps** +- Add the needed config in the `cdb_conf` table: + - `redis_metadata_config` and `redis_metrics_conf` + - `{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}` + - `heremaps_conf` + - `{"app_id": "APP_ID", "app_code": "APP_CODE"}` +- Deploy the client to all the servers with the new version +- Deploy the editor with the new geocoder api version changed (https://github.com/CartoDB/cartodb/blob/master/app/models/user/db_service.rb#L18) +- Execute the rails task to update first the CartoDB team organizaton to test in production + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['team']` +- Check if all works perfectly for our team. If so, execute the rake tasks to update all the users and organizations: + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['', true]` + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users['', true]` From 4e53397f1ce16a06e2f055ff9e66128d058326cd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 25 Jan 2016 17:36:18 +0100 Subject: [PATCH 39/40] Change heremaps module to support host as parameter --- server/extension/sql/0.1.0/20_geocode_street.sql | 2 +- server/lib/python/heremaps/heremaps/heremapsgeocoder.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 3eac170..b396ef3 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(usern RETURNS Geometry AS $$ plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quota_nullable(orgname))) + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] if user_geocoder_config.heremaps_geocoder: diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index 0e7852d..9ebe1df 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -13,8 +13,8 @@ from heremaps.heremapsexceptions import MalformedResult class Geocoder: 'A Here Maps Geocoder wrapper for python' - URL_GEOCODE_JSON = 'http://geocoder.api.here.com/6.2/geocode.json' - URL_DEV_GEOCODE_JSON = 'http://localhost:6083/geocode.json' + PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' + STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json' DEFAULT_MAXRESULTS = 1 DEFAULT_GEN = 9 @@ -55,11 +55,12 @@ class Geocoder: maxresults = '' def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, - gen=DEFAULT_GEN): + gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): self.app_id = app_id self.app_code = app_code self.maxresults = maxresults self.gen = gen + self.host = host def geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): @@ -80,7 +81,7 @@ class Geocoder: 'gen': self.gen } request_params.update(params) - response = requests.get(self.URL_GEOCODE_JSON, params=request_params) + response = requests.get(self.host, params=request_params) if response.status_code == requests.codes.ok: return json.loads(response.text) else: From 593da7926a030ada3e552cea6f7f2fca53f7212b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 25 Jan 2016 17:57:43 +0100 Subject: [PATCH 40/40] Remove duplicated old rule --- server/extension/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/extension/Makefile b/server/extension/Makefile index f42de4a..c3f5de8 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -29,10 +29,6 @@ SOURCES_DATA_DIR = sql/$(EXTVERSION) SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) -$(DATA): $(SOURCES_DATA) - rm -f $@ - cat $(SOURCES_DATA) >> $@ - $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@