User config now is retrieved from redis
This commit is contained in:
Mario de Frutos
parent
a7371c0a48
commit
b2fa8739e2
@ -1,4 +1,5 @@
|
||||
-- Get the Redis configuration from the _conf table --
|
||||
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf();
|
||||
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text)
|
||||
RETURNS cdb_geocoder_server._redis_conf_params AS $$
|
||||
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
|
||||
@ -20,7 +21,8 @@ $$ LANGUAGE plpythonu;
|
||||
-- Get the connection to redis from cache or create a new one
|
||||
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
|
||||
RETURNS boolean AS $$
|
||||
if user_id in GD and 'redis_connection' in GD[user_id]:
|
||||
cache_key = "redis_connection_{0}".format(user_id)
|
||||
if cache_key in GD:
|
||||
return False
|
||||
else:
|
||||
from cartodb_geocoder import redis_helper
|
||||
@ -40,7 +42,7 @@ RETURNS boolean AS $$
|
||||
metrics_config_params['sentinel_master_id'],
|
||||
timeout=metrics_config_params['timeout'],
|
||||
redis_db=metrics_config_params['redis_db']).redis_connection()
|
||||
GD[user_id] = {
|
||||
GD[cache_key] = {
|
||||
'redis_metadata_connection': redis_metadata_connection,
|
||||
'redis_metrics_connection': redis_metrics_connection,
|
||||
}
|
||||
|
||||
@ -1,104 +1,130 @@
|
||||
import json
|
||||
from dateutil.parser import parse as date_parse
|
||||
|
||||
|
||||
class ConfigException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UserConfig:
|
||||
|
||||
USER_CONFIG_KEYS = ['is_organization', 'entity_name']
|
||||
USER_CONFIG_KEYS = ['is_organization', 'entity_name']
|
||||
|
||||
def __init__(self, user_config_json, db_user_id = None):
|
||||
config = json.loads(user_config_json)
|
||||
filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() }
|
||||
self.__check_config(filtered_config)
|
||||
self.__parse_config(filtered_config)
|
||||
self._user_id = self.__extract_uuid(db_user_id)
|
||||
def __init__(self, user_config_json, db_user_id=None):
|
||||
config = json.loads(user_config_json)
|
||||
filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()}
|
||||
self.__check_config(filtered_config)
|
||||
self.__parse_config(filtered_config)
|
||||
|
||||
def __check_config(self, filtered_config):
|
||||
if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS):
|
||||
raise ConfigException("Passed user configuration is not correct, check it please")
|
||||
def __check_config(self, filtered_config):
|
||||
if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS):
|
||||
raise ConfigException(
|
||||
"Passed user configuration is not correct, check it please")
|
||||
|
||||
return True
|
||||
return True
|
||||
|
||||
@property
|
||||
def is_organization(self):
|
||||
return self._is_organization
|
||||
@property
|
||||
def is_organization(self):
|
||||
return self._is_organization
|
||||
|
||||
@property
|
||||
def entity_name(self):
|
||||
return self._entity_name
|
||||
@property
|
||||
def entity_name(self):
|
||||
return self._entity_name
|
||||
|
||||
@property
|
||||
def user_id(self):
|
||||
return self._user_id
|
||||
def __parse_config(self, filtered_config):
|
||||
self._is_organization = filtered_config['is_organization']
|
||||
self._entity_name = filtered_config['entity_name']
|
||||
|
||||
def __parse_config(self, filtered_config):
|
||||
self._is_organization = filtered_config['is_organization']
|
||||
self._entity_name = filtered_config['entity_name']
|
||||
|
||||
def __extract_uuid(self, db_user_id):
|
||||
# Format: development_cartodb_user_<UUID>
|
||||
return db_user_id.split('_')[-1]
|
||||
|
||||
class GeocoderConfig:
|
||||
|
||||
GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key',
|
||||
'nokia_monthly_quota', 'nokia_soft_geocoder_limit']
|
||||
NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit']
|
||||
NOKIA_GEOCODER = 'nokia'
|
||||
NOKIA_QUOTA_KEY = 'nokia_monthly_quota'
|
||||
NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit'
|
||||
GOOGLE_GEOCODER = 'google'
|
||||
GEOCODER_TYPE = 'street_geocoder_provider'
|
||||
GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key'
|
||||
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
|
||||
'geocoding_quota', 'soft_geocoding_limit',
|
||||
'geocoder_type', 'period_end_date']
|
||||
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
|
||||
NOKIA_GEOCODER = 'heremaps'
|
||||
GOOGLE_GEOCODER = 'google'
|
||||
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
|
||||
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
|
||||
GEOCODER_TYPE = 'geocoder_type'
|
||||
QUOTA_KEY = 'geocoding_quota'
|
||||
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
|
||||
PERIOD_END_DATE = 'period_end_date'
|
||||
|
||||
def __init__(self, geocoder_config_json):
|
||||
config = json.loads(geocoder_config_json)
|
||||
filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() }
|
||||
self.__check_config(filtered_config)
|
||||
self.__parse_config(filtered_config)
|
||||
def __init__(self, redis_connection, username, orgname=None):
|
||||
self._redis_connection = redis_connection
|
||||
config = self.__get_user_config(username, orgname)
|
||||
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
|
||||
self.__check_config(filtered_config)
|
||||
self.__parse_config(filtered_config)
|
||||
|
||||
def __check_config(self, filtered_config):
|
||||
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
||||
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
|
||||
raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""")
|
||||
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
||||
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
||||
raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'")
|
||||
def __get_user_config(self, username, orgname=None):
|
||||
user_config = self._redis_connection.hgetall(
|
||||
"rails:users:{0}".format(username))
|
||||
if orgname:
|
||||
org_config = self._redis_connection.hgetall(
|
||||
"rails:orgs:{0}".format(orgname))
|
||||
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
|
||||
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
|
||||
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
||||
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
|
||||
return True
|
||||
return user_config
|
||||
|
||||
def __parse_config(self, filtered_config):
|
||||
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
|
||||
self._google_maps_private_key = None
|
||||
self._nokia_monthly_quota = 0
|
||||
self._nokia_soft_geocoder_limit = False
|
||||
if self.GOOGLE_GEOCODER == self._geocoder_type:
|
||||
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
elif self.NOKIA_GEOCODER == self._geocoder_type:
|
||||
self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY]
|
||||
self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY]
|
||||
def __check_config(self, filtered_config):
|
||||
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
||||
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
|
||||
raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""")
|
||||
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
||||
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
||||
raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'")
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
return self._geocoder_type
|
||||
return True
|
||||
|
||||
@property
|
||||
def nokia_geocoder(self):
|
||||
return self._geocoder_type == self.NOKIA_GEOCODER
|
||||
def __parse_config(self, filtered_config):
|
||||
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
|
||||
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
||||
self._google_maps_private_key = None
|
||||
self._nokia_monthly_quota = 0
|
||||
self._nokia_soft_geocoder_limit = False
|
||||
if self.GOOGLE_GEOCODER == self._geocoder_type:
|
||||
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
||||
elif self.NOKIA_GEOCODER == self._geocoder_type:
|
||||
self._geocoding_quota = filtered_config[self.QUOTA_KEY]
|
||||
self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY]
|
||||
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
return self._geocoder_type == self.GOOGLE_GEOCODER
|
||||
@property
|
||||
def service_type(self):
|
||||
if self._geocoder_type == self.GOOGLE_GEOCODER:
|
||||
return 'geocoder_google'
|
||||
else:
|
||||
return 'geocoder_here'
|
||||
|
||||
@property
|
||||
def google_api_key(self):
|
||||
return self._google_maps_private_key
|
||||
@property
|
||||
def heremaps_geocoder(self):
|
||||
return self._geocoder_type == self.NOKIA_GEOCODER
|
||||
|
||||
@property
|
||||
def nokia_monthly_quota(self):
|
||||
return self._nokia_monthly_quota
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
return self._geocoder_type == self.GOOGLE_GEOCODER
|
||||
|
||||
@property
|
||||
def nokia_soft_limit(self):
|
||||
return self._nokia_soft_geocoder_limit
|
||||
@property
|
||||
def google_client_id(self):
|
||||
return self._google_maps_client_id
|
||||
|
||||
@property
|
||||
def google_api_key(self):
|
||||
return self._google_maps_private_key
|
||||
|
||||
@property
|
||||
def geocoding_quota(self):
|
||||
return self._geocoding_quota
|
||||
|
||||
@property
|
||||
def soft_geocoding_limit(self):
|
||||
return self._soft_geocoding_limit
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@ -1,20 +1,29 @@
|
||||
from redis.sentinel import Sentinel
|
||||
|
||||
|
||||
class RedisHelper:
|
||||
|
||||
REDIS_DEFAULT_USER_DB = 5
|
||||
REDIS_TIMEOUT = 2 #seconds
|
||||
REDIS_DEFAULT_USER_DB = 5
|
||||
REDIS_DEFAULT_TIMEOUT = 2 #seconds
|
||||
REDIS_SENTINEL_DEFAULT_PORT = 26379
|
||||
|
||||
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
|
||||
self.sentinel_host = sentinel_host
|
||||
self.sentinel_port = sentinel_port
|
||||
self.sentinel_master_id = sentinel_master_id
|
||||
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT
|
||||
self.redis_db = redis_db
|
||||
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
|
||||
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
|
||||
self.sentinel_host = sentinel_host
|
||||
self.sentinel_port = sentinel_port
|
||||
self.sentinel_master_id = sentinel_master_id
|
||||
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
|
||||
self.redis_db = redis_db
|
||||
|
||||
def redis_connection(self):
|
||||
return self.__create_redis_connection()
|
||||
def redis_connection(self):
|
||||
return self.__create_redis_connection()
|
||||
|
||||
def __create_redis_connection(self):
|
||||
sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout)
|
||||
return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db)
|
||||
def __create_redis_connection(self):
|
||||
sentinel = Sentinel([(self.sentinel_host,
|
||||
self.REDIS_SENTINEL_DEFAULT_PORT)],
|
||||
socket_timeout=self.timeout)
|
||||
return sentinel.master_for(
|
||||
self.sentinel_master_id,
|
||||
socket_timeout=self.timeout,
|
||||
db=self.redis_db
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user