diff --git a/server/extension/sql/0.1.0/10_redis_helper.sql b/server/extension/sql/0.1.0/10_redis_helper.sql index c94c508..5d7a43e 100644 --- a/server/extension/sql/0.1.0/10_redis_helper.sql +++ b/server/extension/sql/0.1.0/10_redis_helper.sql @@ -1,4 +1,5 @@ -- Get the Redis configuration from the _conf table -- +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf(); CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text) RETURNS cdb_geocoder_server._redis_conf_params AS $$ conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) @@ -20,7 +21,8 @@ $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) RETURNS boolean AS $$ - if user_id in GD and 'redis_connection' in GD[user_id]: + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: return False else: from cartodb_geocoder import redis_helper @@ -40,7 +42,7 @@ RETURNS boolean AS $$ metrics_config_params['sentinel_master_id'], timeout=metrics_config_params['timeout'], redis_db=metrics_config_params['redis_db']).redis_connection() - GD[user_id] = { + GD[cache_key] = { 'redis_metadata_connection': redis_metadata_connection, 'redis_metrics_connection': redis_metrics_connection, } diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index a2b148f..5914fe9 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -1,104 +1,130 @@ import json +from dateutil.parser import parse as date_parse + class ConfigException(Exception): pass + class UserConfig: - USER_CONFIG_KEYS = ['is_organization', 'entity_name'] + USER_CONFIG_KEYS = ['is_organization', 'entity_name'] - def __init__(self, user_config_json, db_user_id = None): - config = json.loads(user_config_json) - filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) - self._user_id = self.__extract_uuid(db_user_id) + def __init__(self, user_config_json, db_user_id=None): + config = json.loads(user_config_json) + filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()} + self.__check_config(filtered_config) + self.__parse_config(filtered_config) - def __check_config(self, filtered_config): - if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): - raise ConfigException("Passed user configuration is not correct, check it please") + def __check_config(self, filtered_config): + if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): + raise ConfigException( + "Passed user configuration is not correct, check it please") - return True + return True - @property - def is_organization(self): - return self._is_organization + @property + def is_organization(self): + return self._is_organization - @property - def entity_name(self): - return self._entity_name + @property + def entity_name(self): + return self._entity_name - @property - def user_id(self): - return self._user_id + def __parse_config(self, filtered_config): + self._is_organization = filtered_config['is_organization'] + self._entity_name = filtered_config['entity_name'] - def __parse_config(self, filtered_config): - self._is_organization = filtered_config['is_organization'] - self._entity_name = filtered_config['entity_name'] - - def __extract_uuid(self, db_user_id): - # Format: development_cartodb_user_ - return db_user_id.split('_')[-1] class GeocoderConfig: - GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', - 'nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER = 'nokia' - NOKIA_QUOTA_KEY = 'nokia_monthly_quota' - NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit' - GOOGLE_GEOCODER = 'google' - GEOCODER_TYPE = 'street_geocoder_provider' - GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key' + GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', + 'geocoding_quota', 'soft_geocoding_limit', + 'geocoder_type', 'period_end_date'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] + NOKIA_GEOCODER = 'heremaps' + GOOGLE_GEOCODER = 'google' + GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' + GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' + GEOCODER_TYPE = 'geocoder_type' + QUOTA_KEY = 'geocoding_quota' + SOFT_LIMIT_KEY = 'soft_geocoding_limit' + PERIOD_END_DATE = 'period_end_date' - def __init__(self, geocoder_config_json): - config = json.loads(geocoder_config_json) - filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) + def __init__(self, redis_connection, username, orgname=None): + self._redis_connection = redis_connection + config = self.__get_user_config(username, orgname) + filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} + self.__check_config(filtered_config) + self.__parse_config(filtered_config) - def __check_config(self, filtered_config): - if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): - raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""") - elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: - if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): - raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") + def __get_user_config(self, username, orgname=None): + user_config = self._redis_connection.hgetall( + "rails:users:{0}".format(username)) + if orgname: + org_config = self._redis_connection.hgetall( + "rails:orgs:{0}".format(orgname)) + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] + user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] - return True + return user_config - def __parse_config(self, filtered_config): - self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() - self._google_maps_private_key = None - self._nokia_monthly_quota = 0 - self._nokia_soft_geocoder_limit = False - if self.GOOGLE_GEOCODER == self._geocoder_type: - self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] - elif self.NOKIA_GEOCODER == self._geocoder_type: - self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY] - self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY] + def __check_config(self, filtered_config): + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): + raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""") + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): + raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") - @property - def service_type(self): - return self._geocoder_type + return True - @property - def nokia_geocoder(self): - return self._geocoder_type == self.NOKIA_GEOCODER + def __parse_config(self, filtered_config): + self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + self._google_maps_private_key = None + self._nokia_monthly_quota = 0 + self._nokia_soft_geocoder_limit = False + if self.GOOGLE_GEOCODER == self._geocoder_type: + self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] + elif self.NOKIA_GEOCODER == self._geocoder_type: + self._geocoding_quota = filtered_config[self.QUOTA_KEY] + self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY] - @property - def google_geocoder(self): - return self._geocoder_type == self.GOOGLE_GEOCODER + @property + def service_type(self): + if self._geocoder_type == self.GOOGLE_GEOCODER: + return 'geocoder_google' + else: + return 'geocoder_here' - @property - def google_api_key(self): - return self._google_maps_private_key + @property + def heremaps_geocoder(self): + return self._geocoder_type == self.NOKIA_GEOCODER - @property - def nokia_monthly_quota(self): - return self._nokia_monthly_quota + @property + def google_geocoder(self): + return self._geocoder_type == self.GOOGLE_GEOCODER - @property - def nokia_soft_limit(self): - return self._nokia_soft_geocoder_limit \ No newline at end of file + @property + def google_client_id(self): + return self._google_maps_client_id + + @property + def google_api_key(self): + return self._google_maps_private_key + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py index 27d4800..0c686bf 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -1,20 +1,29 @@ from redis.sentinel import Sentinel + class RedisHelper: - REDIS_DEFAULT_USER_DB = 5 - REDIS_TIMEOUT = 2 #seconds + REDIS_DEFAULT_USER_DB = 5 + REDIS_DEFAULT_TIMEOUT = 2 #seconds + REDIS_SENTINEL_DEFAULT_PORT = 26379 - def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.sentinel_host = sentinel_host - self.sentinel_port = sentinel_port - self.sentinel_master_id = sentinel_master_id - self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT - self.redis_db = redis_db + def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, + redis_db=REDIS_DEFAULT_USER_DB, **kwargs): + self.sentinel_host = sentinel_host + self.sentinel_port = sentinel_port + self.sentinel_master_id = sentinel_master_id + self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT + self.redis_db = redis_db - def redis_connection(self): - return self.__create_redis_connection() + def redis_connection(self): + return self.__create_redis_connection() - def __create_redis_connection(self): - sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout) - return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db) \ No newline at end of file + def __create_redis_connection(self): + sentinel = Sentinel([(self.sentinel_host, + self.REDIS_SENTINEL_DEFAULT_PORT)], + socket_timeout=self.timeout) + return sentinel.master_for( + self.sentinel_master_id, + socket_timeout=self.timeout, + db=self.redis_db + )