cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

User config now is retrieved from redis

Browse Source
This commit is contained in:
Mario de Frutos 2016-01-20 16:52:55 +01:00
parent a7371c0a48
commit b2fa8739e2
3 changed files with 129 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/extension/sql/0.1.0/10_redis_helper.sql
View File

@ -1,4 +1,5 @@
-- Get the Redis configuration from the _conf table --
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf();
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
@ -20,7 +21,8 @@ $$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
if user_id in GD and 'redis_connection' in GD[user_id]:
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
@ -40,7 +42,7 @@ RETURNS boolean AS $$
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[user_id] = {
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}

82
server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py
View File

@ -1,8 +1,11 @@
import json
from dateutil.parser import parse as date_parse
class ConfigException(Exception):
pass
class UserConfig:
USER_CONFIG_KEYS = ['is_organization', 'entity_name']
@ -12,11 +15,11 @@ class UserConfig:
filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()}
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
self._user_id = self.__extract_uuid(db_user_id)
def __check_config(self, filtered_config):
if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS):
raise ConfigException("Passed user configuration is not correct, check it please")
raise ConfigException(
"Passed user configuration is not correct, check it please")
return True
@ -28,40 +31,50 @@ class UserConfig:
def entity_name(self):
return self._entity_name
@property
def user_id(self):
return self._user_id
def __parse_config(self, filtered_config):
self._is_organization = filtered_config['is_organization']
self._entity_name = filtered_config['entity_name']
def __extract_uuid(self, db_user_id):
# Format: development_cartodb_user_<UUID>
return db_user_id.split('_')[-1]
class GeocoderConfig:
GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key',
'nokia_monthly_quota', 'nokia_soft_geocoder_limit']
NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit']
NOKIA_GEOCODER = 'nokia'
NOKIA_QUOTA_KEY = 'nokia_monthly_quota'
NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit'
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date']
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
NOKIA_GEOCODER = 'heremaps'
GOOGLE_GEOCODER = 'google'
GEOCODER_TYPE = 'street_geocoder_provider'
GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key'
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
GEOCODER_TYPE = 'geocoder_type'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, geocoder_config_json):
config = json.loads(geocoder_config_json)
def __init__(self, redis_connection, username, orgname=None):
self._redis_connection = redis_connection
config = self.__get_user_config(username, orgname)
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __get_user_config(self, username, orgname=None):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if orgname:
org_config = self._redis_connection.hgetall(
"rails:orgs:{0}".format(orgname))
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
return user_config
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""")
raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'")
@ -70,35 +83,48 @@ class GeocoderConfig:
def __parse_config(self, filtered_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._google_maps_private_key = None
self._nokia_monthly_quota = 0
self._nokia_soft_geocoder_limit = False
if self.GOOGLE_GEOCODER == self._geocoder_type:
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
elif self.NOKIA_GEOCODER == self._geocoder_type:
self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY]
self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY]
self._geocoding_quota = filtered_config[self.QUOTA_KEY]
self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY]
@property
def service_type(self):
return self._geocoder_type
if self._geocoder_type == self.GOOGLE_GEOCODER:
return 'geocoder_google'
else:
return 'geocoder_here'
@property
def nokia_geocoder(self):
def heremaps_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@property
def google_api_key(self):
return self._google_maps_private_key
@property
def nokia_monthly_quota(self):
return self._nokia_monthly_quota
def geocoding_quota(self):
return self._geocoding_quota
@property
def nokia_soft_limit(self):
return self._nokia_soft_geocoder_limit
def soft_geocoding_limit(self):
return self._soft_geocoding_limit
@property
def period_end_date(self):
return self._period_end_date

19
server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py
View File

@ -1,20 +1,29 @@
from redis.sentinel import Sentinel
class RedisHelper:
REDIS_DEFAULT_USER_DB = 5
REDIS_TIMEOUT = 2 #seconds
REDIS_DEFAULT_TIMEOUT = 2 #seconds
REDIS_SENTINEL_DEFAULT_PORT = 26379
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.sentinel_host = sentinel_host
self.sentinel_port = sentinel_port
self.sentinel_master_id = sentinel_master_id
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def redis_connection(self):
return self.__create_redis_connection()
def __create_redis_connection(self):
sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout)
return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db)
sentinel = Sentinel([(self.sentinel_host,
self.REDIS_SENTINEL_DEFAULT_PORT)],
socket_timeout=self.timeout)
return sentinel.master_for(
self.sentinel_master_id,
socket_timeout=self.timeout,
db=self.redis_db
)