Merge pull request #284 from CartoDB/redis-refactor-take2

Redis refactor: Take 2

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 *.pyc
+.coverage
 cartodb_services.egg-info/
 build/
 dist/

server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql

@@ -0,0 +1,69 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.16.0'" to load this file. \quit
+
+-- Here goes your code to upgrade/downgrade
+
+-- This is done in order to avoid an undesired depedency on cartodb extension
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
+RETURNS JSON AS $$
+    SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
+$$ LANGUAGE SQL STABLE SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger
+  from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
+  from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+  from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
+  from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+  from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
+  from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
+  from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
+
+  server_config_backend = ServerConfigBackendFactory().get()
+  environment = ServerEnvironmentBuilder(server_config_backend).get()
+  user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
+  org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
+
+  logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
+  logger = Logger(logger_config)
+
+  mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
+
+  redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
+
+  quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3, search_type='address')
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using mapzen')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql

@@ -0,0 +1,54 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.1'" to load this file. \quit
+
+-- Here goes your code to upgrade/downgrade
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_conf_getconf(text);
+
+-- Geocodes a street address given a searchtext and a state and/or country
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  from cartodb_services.config.user import User
+  from cartodb_services.config.configs import ConfigsFactory
+  from cartodb_services.config.hires_geocoder_config import HiResGeocoderConfigFactory
+  from cartodb_services.request.request import RequestFactory
+
+  user = User(username, orgname)
+  configs = ConfigsFactory.get(user)
+  request = RequestFactory().create(user, configs, 'cdb_geocode_street_point')
+
+  # TODO change to hires_geocoder_config = HiResGeocoderConfigFactory.get(request)
+  hires_geocoder_config = HiResGeocoderConfigFactory(configs).get(user)
+
+  if hires_geocoder_config.provider == 'here':
+    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif hires_geocoder_config.provider == 'google':
+    google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif hires_geocoder_config.provider == 'mapzen':
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    raise Exception('Requested geocoder is not available')
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  if user_geocoder_config.heremaps_geocoder:
+    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    raise Exception('Here geocoder is not available for your account.')
+
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.15.1'
+default_version = '0.16.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/15_config_helper.sql

@@ -10,6 +10,12 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
+-- This is done in order to avoid an undesired depedency on cartodb extension
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
+RETURNS JSON AS $$
+    SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
+$$ LANGUAGE SQL STABLE SECURITY DEFINER;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
 RETURNS boolean AS $$
   cache_key = "user_geocoder_config_{0}".format(username)

server/extension/sql/20_geocode_street.sql

@@ -137,23 +137,38 @@ $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
   from cartodb_services.mapzen import MapzenGeocoder
   from cartodb_services.mapzen.types import country_to_iso3
   from cartodb_services.metrics import QuotaService
-  from cartodb_services.tools import Logger,LoggerConfig
+  from cartodb_services.tools import Logger
+  from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
+  from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+  from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
+  from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+  from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
+  from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
+  from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
 
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  server_config_backend = ServerConfigBackendFactory().get()
+  environment = ServerEnvironmentBuilder(server_config_backend).get()
+  user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
+  org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
 
-  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
-  logger_config = GD["logger_config"]
+  logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
   logger = Logger(logger_config)
-  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
+
+  redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
+
+  quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
   try:
-    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+    geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
     country_iso3 = None
     if country:
       country_iso3 = country_to_iso3(country)

server/lib/python/cartodb_services/cartodb_services/__init__.py

@@ -0,0 +1,35 @@
+# NOTE: This init function must be called from plpythonu entry points to
+# initialize cartodb_services module properly. E.g:
+#
+# CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(...)
+# RETURNS SETOF cdb_dataservices_server.isoline AS $$
+#
+#     import cartodb_services
+#     cartodb_services.init(plpy, GD)
+#
+#     # rest of the code here
+#     cartodb_services.GD[key] = val
+#     cartodb_services.plpy.execute('SELECT * FROM ...')
+#
+# $$ LANGUAGE plpythonu;
+
+plpy = None
+GD = None
+
+def init(_plpy, _GD):
+    global plpy
+    global GD
+
+    if plpy is None:
+        plpy = _plpy
+
+    if GD is None:
+        GD = _GD
+
+def _reset():
+    # NOTE: just for testing
+    global plpy
+    global GD
+
+    plpy = None
+    GD = None

server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py

@@ -0,0 +1,24 @@
+from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder
+from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder
+from cartodb_services.refactor.storage.redis_config import RedisOrgConfigStorageBuilder
+
+class OrgConfigBackendFactory(object):
+    """
+    This class abstracts the creation of an org configuration backend. It will return
+    an implementation of the ConfigBackendInterface appropriate to the org, depending
+    on the environment.
+    """
+
+    def __init__(self, orgname, environment, server_config_backend):
+        self._orgname = orgname
+        self._environment = environment
+        self._server_config_backend = server_config_backend
+
+    def get(self):
+        if self._environment.is_onpremise:
+            org_config_backend = self._server_config_backend
+        else:
+            redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_backend).get()
+            redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get()
+            org_config_backend = RedisOrgConfigStorageBuilder(redis_metadata_connection, self._orgname).get()
+        return org_config_backend

server/lib/python/cartodb_services/cartodb_services/refactor/backend/redis_metrics_connection.py

@@ -0,0 +1,17 @@
+from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock
+from cartodb_services.refactor.storage.redis_connection_config import RedisMetricsConnectionConfigBuilder
+from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder
+
+class RedisMetricsConnectionFactory(object):
+    def __init__(self, environment, server_config_storage):
+        self._environment = environment
+        self._server_config_storage = server_config_storage
+
+    def get(self):
+        if self._environment.is_onpremise:
+           redis_metrics_connection = RedisConnectionMock()
+        else:
+          redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(self._server_config_storage).get()
+          redis_metrics_connection = RedisConnectionBuilder(redis_metrics_connection_config).get()
+        return redis_metrics_connection
+

server/lib/python/cartodb_services/cartodb_services/refactor/backend/server_config.py

@@ -0,0 +1,13 @@
+from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage
+
+
+class ServerConfigBackendFactory(object):
+    """
+    This class creates a backend to retrieve server configurations (implementing the ConfigBackendInterface).
+
+    At this moment it will always return an InDbServerConfigStorage, but nothing prevents from changing the
+    implementation. To something that reads from a file, memory or whatever. It is mostly there to keep
+    the layers separated.
+    """
+    def get(self):
+        return InDbServerConfigStorage()

server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py

@@ -0,0 +1,24 @@
+from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder
+from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder
+from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorageBuilder
+
+class UserConfigBackendFactory(object):
+    """
+    This class abstracts the creation of a user configuration backend. It will return
+    an implementation of the ConfigBackendInterface appropriate to the user, depending
+    on the environment.
+    """
+
+    def __init__(self, username, environment, server_config_backend):
+        self._username = username
+        self._environment = environment
+        self._server_config_backend = server_config_backend
+
+    def get(self):
+        if self._environment.is_onpremise:
+            user_config_backend = self._server_config_backend
+        else:
+            redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_backend).get()
+            redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get()
+            user_config_backend = RedisUserConfigStorageBuilder(redis_metadata_connection, self._username).get()
+        return user_config_backend

server/lib/python/cartodb_services/cartodb_services/refactor/config/exceptions.py

@@ -0,0 +1,2 @@
+class ConfigException(Exception):
+    pass

server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py

@@ -0,0 +1,57 @@
+class ServerEnvironment(object):
+
+    DEVELOPMENT = 'development'
+    STAGING = 'staging'
+    PRODUCTION = 'production'
+    ONPREMISE = 'onpremise'
+
+    VALID_ENVIRONMENTS = [
+        DEVELOPMENT,
+        STAGING,
+        PRODUCTION,
+        ONPREMISE
+    ]
+
+    def __init__(self, environment_str):
+        assert environment_str in self.VALID_ENVIRONMENTS
+        self._environment_str = environment_str
+
+    def __str__(self):
+        return self._environment_str
+
+    @property
+    def is_development(self):
+        return self._environment_str == self.DEVELOPMENT
+
+    @property
+    def is_staging(self):
+        return self._environment_str == self.STAGING
+
+    @property
+    def is_production(self):
+        return self._environment_str == self.PRODUCTION
+
+    @property
+    def is_onpremise(self):
+        return self._environment_str == self.ONPREMISE
+
+    def __eq__(self, other):
+        return self._environment_str == other._environment_str
+
+
+class ServerEnvironmentBuilder(object):
+
+    DEFAULT_ENVIRONMENT = ServerEnvironment.DEVELOPMENT
+
+    def __init__(self, server_config_storage):
+        self._server_config_storage = server_config_storage
+
+    def get(self):
+        server_config = self._server_config_storage.get('server_conf')
+
+        if not server_config or 'environment' not in server_config:
+            environment_str = self.DEFAULT_ENVIRONMENT
+        else:
+            environment_str = server_config['environment']
+
+        return ServerEnvironment(environment_str)

server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py

@@ -0,0 +1,11 @@
+import abc
+
+class ConfigBackendInterface(object):
+    """This is an interface that all config backends must abide to"""
+
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def get(self, key):
+        """Return a value based on the key supplied from some storage"""
+        pass

server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py

@@ -0,0 +1,112 @@
+from dateutil.parser import parse as date_parse
+
+class MapzenGeocoderConfig(object):
+    """
+    Value object that represents the configuration needed to operate the mapzen service.
+    """
+
+    def __init__(self,
+                 geocoding_quota,
+                 soft_geocoding_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 mapzen_api_key,
+                 username,
+                 organization):
+        self._geocoding_quota = geocoding_quota
+        self._soft_geocoding_limit = soft_geocoding_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._mapzen_api_key = mapzen_api_key
+        self._username = username
+        self._organization = organization
+
+    # Kind of generic properties. Note which ones are for actually running the
+    # service and which ones are needed for quota stuff.
+    @property
+    def service_type(self):
+        return 'geocoder_mapzen'
+
+    @property
+    def provider(self):
+        return 'mapzen'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    # Server config, TODO: locate where this is actually used
+    @property
+    def log_path(self):
+        return self._log_path
+
+    # This is actually the specific one to run requests against the remote endpoitn
+    @property
+    def mapzen_api_key(self):
+        return self._mapzen_api_key
+
+    # These two identify the user
+    @property
+    def username(self):
+        return self._username
+    @property
+    def organization(self):
+        return self._organization
+
+    # TODO: for BW compat, remove
+    @property
+    def google_geocoder(self):
+        return False
+
+
+class MapzenGeocoderConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+
+
+    def get(self):
+        mapzen_server_conf = self._server_conf.get('mapzen_conf')
+        geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota']
+        mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
+
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit')
+
+        cost_per_hit=0
+
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf['geocoder_log_path']
+
+        return MapzenGeocoderConfig(geocoding_quota,
+                                    soft_geocoding_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    mapzen_api_key,
+                                    self._username,
+                                    self._orgname)

server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py

@@ -0,0 +1,12 @@
+from ..core.interfaces import ConfigBackendInterface
+
+class InMemoryConfigStorage(ConfigBackendInterface):
+
+    def __init__(self, config_hash={}):
+        self._config_hash = config_hash
+
+    def get(self, key):
+        try:
+            return self._config_hash[key]
+        except KeyError:
+            return None

server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py

@@ -0,0 +1,6 @@
+from ..core.interfaces import ConfigBackendInterface
+
+class NullConfigStorage(ConfigBackendInterface):
+
+    def get(self, key):
+        return None

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -0,0 +1,36 @@
+from ..core.interfaces import ConfigBackendInterface
+from null_config import NullConfigStorage
+
+
+class RedisConfigStorage(ConfigBackendInterface):
+
+    def __init__(self, connection, config_key):
+        self._connection = connection
+        self._config_key = config_key
+        self._data = None
+
+    def get(self, key):
+        if not self._data:
+            self._data = self._connection.hgetall(self._config_key)
+        return self._data[key]
+
+
+class RedisUserConfigStorageBuilder(object):
+    def __init__(self, redis_connection, username):
+        self._redis_connection = redis_connection
+        self._username = username
+
+    def get(self):
+        return RedisConfigStorage(self._redis_connection, 'rails:users:{0}'.format(self._username))
+
+
+class RedisOrgConfigStorageBuilder(object):
+    def __init__(self, redis_connection, orgname):
+        self._redis_connection = redis_connection
+        self._orgname = orgname
+
+    def get(self):
+        if self._orgname:
+            return RedisConfigStorage(self._redis_connection, 'rails:orgs:{0}'.format(self._orgname))
+        else:
+            return NullConfigStorage()

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py

@@ -0,0 +1,22 @@
+from redis.sentinel import Sentinel
+from redis import StrictRedis
+
+class RedisConnectionBuilder():
+
+    def __init__(self, connection_config):
+        self._config = connection_config
+
+    def get(self):
+        if self._config.sentinel_id:
+            sentinel = Sentinel([(self._config.host,
+                                  self._config.port)],
+                                socket_timeout=self._config.timeout)
+            return sentinel.master_for(self._config.sentinel_id,
+                                       socket_timeout=self._config.timeout,
+                                       db=self._config.db,
+                                       retry_on_timeout=True)
+        else:
+            conn = StrictRedis(host=self._config.host, port=self._config.port,
+                               db=self._config.db, retry_on_timeout=True,
+                               socket_timeout=self._config.timeout)
+            return conn

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py

@@ -0,0 +1,80 @@
+from cartodb_services.refactor.config.exceptions import ConfigException
+from abc import ABCMeta, abstractmethod
+
+
+class RedisConnectionConfig(object):
+    """
+    This represents a value object to contain configuration needed to set up
+    a connection to a redis server.
+    """
+
+    def __init__(self, host, port, timeout, db, sentinel_id):
+        self._host = host
+        self._port = port
+        self._timeout = timeout
+        self._db = db
+        self._sentinel_id = sentinel_id
+
+    @property
+    def host(self):
+        return self._host
+
+    @property
+    def port(self):
+        return self._port
+
+    @property
+    def timeout(self):
+        return self._timeout
+
+    @property
+    def db(self):
+        return self._db
+
+    @property
+    def sentinel_id(self):
+        return self._sentinel_id
+
+
+class RedisConnectionConfigBuilder(object):
+
+    __metaclass__ = ABCMeta
+
+    DEFAULT_USER_DB = 5
+    DEFAULT_TIMEOUT = 1.5  # seconds
+
+    @abstractmethod
+    def __init__(self, server_config_storage, config_key):
+        self._server_config_storage = server_config_storage
+        self._config_key = config_key
+
+    def get(self):
+        conf = self._server_config_storage.get(self._config_key)
+        if conf is None:
+            raise ConfigException("There is no redis configuration defined")
+
+        host = conf['redis_host']
+        port = conf['redis_port']
+        timeout = conf['timeout'] or self.DEFAULT_TIMEOUT
+        db = conf['redis_db'] or self.DEFAULT_USER_DB
+        sentinel_id = conf['sentinel_master_id']
+
+        return RedisConnectionConfig(host, port, timeout, db, sentinel_id)
+
+
+class RedisMetadataConnectionConfigBuilder(RedisConnectionConfigBuilder):
+
+    def __init__(self, server_config_storage):
+        super(RedisMetadataConnectionConfigBuilder, self).__init__(
+            server_config_storage,
+            'redis_metadata_config'
+        )
+
+
+class RedisMetricsConnectionConfigBuilder(RedisConnectionConfigBuilder):
+
+    def __init__(self, server_config_storage):
+        super(RedisMetricsConnectionConfigBuilder, self).__init__(
+            server_config_storage,
+            'redis_metrics_config'
+        )

server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py

@@ -0,0 +1,14 @@
+import json
+import cartodb_services
+from ..core.interfaces import ConfigBackendInterface
+
+class InDbServerConfigStorage(ConfigBackendInterface):
+
+    def get(self, key):
+        sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key)
+        rows = cartodb_services.plpy.execute(sql, 1)
+        json_output = rows[0]['conf']
+        if json_output:
+            return json.loads(json_output)
+        else:
+            return None

server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py

@@ -0,0 +1,52 @@
+from cartodb_services.refactor.config.exceptions import ConfigException
+
+class LoggerConfig(object):
+
+    """This class is a value object needed to setup a Logger"""
+
+    def __init__(self, server_environment, rollbar_api_key, log_file_path, min_log_level):
+        self._server_environment = server_environment
+        self._rollbar_api_key = rollbar_api_key
+        self._log_file_path = log_file_path
+        self._min_log_level = min_log_level
+
+    @property
+    def environment(self):
+        return self._server_environment
+
+    @property
+    def rollbar_api_key(self):
+        return self._rollbar_api_key
+
+    @property
+    def log_file_path(self):
+        return self._log_file_path
+
+    @property
+    def min_log_level(self):
+        return self._min_log_level
+
+# TODO this needs tests
+class LoggerConfigBuilder(object):
+
+    def __init__(self, environment, server_config_storage):
+        self._server_environment = environment
+        self._server_config_storage = server_config_storage
+
+    def get(self):
+        logger_conf = self._server_config_storage.get('logger_conf')
+        if not logger_conf:
+            raise ConfigException('Logger configuration missing')
+        
+        rollbar_api_key = self._get_value_or_none(logger_conf, 'rollbar_api_key')
+        log_file_path = self._get_value_or_none(logger_conf, 'log_file_path')
+        min_log_level = self._get_value_or_none(logger_conf, 'min_log_level') or 'warning'
+
+        logger_config = LoggerConfig(str(self._server_environment), rollbar_api_key, log_file_path, min_log_level)
+        return logger_config
+
+    def _get_value_or_none(self, logger_conf, key):
+        value = None
+        if key in logger_conf:
+            value = logger_conf[key]
+        return value

server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py

@@ -0,0 +1,8 @@
+class RedisConnectionMock(object):
+    """ Simple class to mock a dummy behaviour for Redis related functions """
+
+    def zscore(self, redis_prefix, day):
+        pass
+
+    def zincrby(self, redis_prefix, day, amount):
+        pass

server/lib/python/cartodb_services/test/refactor/core/test_environment.py

@@ -0,0 +1,47 @@
+from unittest import TestCase
+from cartodb_services.refactor.core.environment import *
+from nose.tools import raises
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+
+class TestServerEnvironment(TestCase):
+
+    def test_can_be_a_valid_one(self):
+        env_dev = ServerEnvironment('development')
+        env_staging = ServerEnvironment('staging')
+        env_prod = ServerEnvironment('production')
+        env_onpremise = ServerEnvironment('onpremise')
+
+    @raises(AssertionError)
+    def test_cannot_be_a_non_valid_one(self):
+        env_whatever = ServerEnvironment('whatever')
+
+    def test_is_on_premise_returns_true_when_onpremise(self):
+        assert ServerEnvironment('onpremise').is_onpremise == True
+
+    def test_is_on_premise_returns_true_when_any_other(self):
+        assert ServerEnvironment('development').is_onpremise == False
+        assert ServerEnvironment('staging').is_onpremise == False
+        assert ServerEnvironment('production').is_onpremise == False
+
+    def test_equality(self):
+        assert ServerEnvironment('development') == ServerEnvironment('development')
+        assert ServerEnvironment('development') <> ServerEnvironment('onpremise')
+
+
+class TestServerEnvironmentBuilder(TestCase):
+
+    def test_returns_env_according_to_configuration(self):
+        server_config_storage = InMemoryConfigStorage({
+            'server_conf': {
+                'environment': 'staging'
+            }
+        })
+        server_env = ServerEnvironmentBuilder(server_config_storage).get()
+        assert server_env.is_staging == True
+
+    def test_returns_default_when_no_server_conf(self):
+        server_config_storage = InMemoryConfigStorage({})
+        server_env = ServerEnvironmentBuilder(server_config_storage).get()
+
+        assert server_env.is_development == True
+        assert str(server_env) == ServerEnvironmentBuilder.DEFAULT_ENVIRONMENT

server/lib/python/cartodb_services/test/refactor/storage/test_mem_config.py

@@ -0,0 +1,12 @@
+from unittest import TestCase
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+
+class TestInMemoryConfigStorage(TestCase):
+
+    def test_can_provide_values_from_hash(self):
+        server_config = InMemoryConfigStorage({'any_key': 'any_value'})
+        assert server_config.get('any_key') == 'any_value'
+
+    def test_gets_none_if_cannot_retrieve_key(self):
+        server_config = InMemoryConfigStorage()
+        assert server_config.get('any_non_existing_key') == None

server/lib/python/cartodb_services/test/refactor/storage/test_null_config.py

@@ -0,0 +1,14 @@
+from unittest import TestCase
+from cartodb_services.refactor.storage.null_config import NullConfigStorage
+from cartodb_services.refactor.core.interfaces import ConfigBackendInterface
+
+
+class TestNullConfigStorage(TestCase):
+
+    def test_is_a_config_backend(self):
+        null_config = NullConfigStorage()
+        assert isinstance(null_config, ConfigBackendInterface)
+
+    def test_returns_none_regardless_of_input(self):
+        null_config = NullConfigStorage()
+        assert null_config.get('whatever') is None

server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py

@@ -0,0 +1,77 @@
+from unittest import TestCase
+from cartodb_services.refactor.storage.redis_config import *
+from mockredis import MockRedis
+from mock import Mock, MagicMock
+from nose.tools import raises
+
+
+class TestRedisConfigStorage(TestCase):
+
+    CONFIG_HASH_KEY = 'mykey'
+
+    def test_can_get_a_config_field(self):
+        connection = MockRedis()
+        connection.hset(self.CONFIG_HASH_KEY, 'field1', 42)
+        redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY)
+
+        value = redis_config.get('field1')
+        assert type(value) == str # this is something to take into account, redis always returns strings
+        assert value == '42'
+
+    @raises(KeyError)
+    def test_raises_an_exception_if_config_key_not_present(self):
+        connection = MockRedis()
+        redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY)
+        redis_config.get('whatever_field')
+
+    @raises(KeyError)
+    def test_returns_nothing_if_field_not_present(self):
+        connection = MockRedis()
+        connection.hmset(self.CONFIG_HASH_KEY, {'field1': 42, 'field2': 43})
+        redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY)
+        redis_config.get('whatever_field')
+
+    def test_it_reads_the_config_hash_just_once(self):
+        connection = Mock()
+        connection.hgetall = MagicMock(return_value={'field1': '42'})
+        redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY)
+
+        assert redis_config.get('field1') == '42'
+        assert redis_config.get('field1') == '42'
+
+        connection.hgetall.assert_called_once_with(self.CONFIG_HASH_KEY)
+
+
+class TestRedisUserConfigStorageBuilder(TestCase):
+
+    USERNAME = 'john'
+    EXPECTED_REDIS_CONFIG_HASH_KEY = 'rails:users:john'
+    
+    def test_it_reads_the_correct_hash_key(self):
+        connection = Mock()
+        connection.hgetall = MagicMock(return_value={'an_user_config_field': 'nice'})
+        redis_config = RedisConfigStorage(connection, self.EXPECTED_REDIS_CONFIG_HASH_KEY)
+
+        redis_config = RedisUserConfigStorageBuilder(connection, self.USERNAME).get()
+        assert redis_config.get('an_user_config_field') == 'nice'
+        connection.hgetall.assert_called_once_with(self.EXPECTED_REDIS_CONFIG_HASH_KEY)
+
+
+class TestRedisOrgConfigStorageBuilder(TestCase):
+
+    ORGNAME = 'smith'
+    EXPECTED_REDIS_CONFIG_HASH_KEY = 'rails:orgs:smith'
+
+    def test_it_reads_the_correct_hash_key(self):
+        connection = Mock()
+        connection.hgetall = MagicMock(return_value={'an_org_config_field': 'awesome'})
+        redis_config = RedisConfigStorage(connection, self.EXPECTED_REDIS_CONFIG_HASH_KEY)
+
+        redis_config = RedisOrgConfigStorageBuilder(connection, self.ORGNAME).get()
+        assert redis_config.get('an_org_config_field') == 'awesome'
+        connection.hgetall.assert_called_once_with(self.EXPECTED_REDIS_CONFIG_HASH_KEY)
+
+    def test_it_returns_a_null_config_storage_if_theres_no_orgname(self):
+        redis_config = RedisOrgConfigStorageBuilder(None, None).get()
+        assert type(redis_config) == NullConfigStorage
+        assert redis_config.get('whatever') == None

server/lib/python/cartodb_services/test/refactor/storage/test_redis_connection_config.py

@@ -0,0 +1,101 @@
+from unittest import TestCase
+from cartodb_services.refactor.storage.redis_connection_config import *
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+from cartodb_services.refactor.config.exceptions import ConfigException
+
+class TestRedisConnectionConfig(TestCase):
+
+    def test_config_holds_values(self):
+        # this is mostly for completeness, dummy class, dummy test
+        config = RedisConnectionConfig('myhost.com', 6379, 0.1, 5, None)
+        assert config.host == 'myhost.com'
+        assert config.port == 6379
+        assert config.timeout == 0.1
+        assert config.db == 5
+        assert config.sentinel_id is None
+
+
+class TestRedisConnectionConfigBuilder(TestCase):
+
+    def test_it_raises_exception_as_it_is_abstract(self):
+        server_config_storage = InMemoryConfigStorage()
+        self.assertRaises(TypeError, RedisConnectionConfigBuilder, server_config_storage, 'whatever_key')
+
+
+class TestRedisMetadataConnectionConfigBuilder(TestCase):
+
+    def test_it_raises_exception_if_config_is_missing(self):
+        server_config_storage = InMemoryConfigStorage()
+        config_builder = RedisMetadataConnectionConfigBuilder(server_config_storage)
+        self.assertRaises(ConfigException, config_builder.get)
+
+    def test_it_gets_a_valid_config_from_the_server_storage(self):
+        server_config_storage = InMemoryConfigStorage({
+            'redis_metadata_config': {
+                'redis_host': 'myhost.com',
+                'redis_port': 6379,
+                'timeout': 0.2,
+                'redis_db': 3,
+                'sentinel_master_id': None
+            }
+        })
+        config = RedisMetadataConnectionConfigBuilder(server_config_storage).get()
+        assert config.host == 'myhost.com'
+        assert config.port == 6379
+        assert config.timeout == 0.2
+        assert config.db == 3
+        assert config.sentinel_id is None
+
+    def test_it_gets_a_default_timeout_if_none(self):
+        server_config_storage = InMemoryConfigStorage({
+            'redis_metadata_config': {
+                'redis_host': 'myhost.com',
+                'redis_port': 6379,
+                'timeout': None,
+                'redis_db': 3,
+                'sentinel_master_id': None
+            }
+        })
+        config = RedisMetadataConnectionConfigBuilder(server_config_storage).get()
+        assert config.host == 'myhost.com'
+        assert config.port == 6379
+        assert config.timeout == RedisConnectionConfigBuilder.DEFAULT_TIMEOUT
+        assert config.db == 3
+        assert config.sentinel_id is None
+
+    def test_it_gets_a_default_db_if_none(self):
+        server_config_storage = InMemoryConfigStorage({
+            'redis_metadata_config': {
+                'redis_host': 'myhost.com',
+                'redis_port': 6379,
+                'timeout': 0.2,
+                'redis_db': None,
+                'sentinel_master_id': None
+            }
+        })
+        config = RedisMetadataConnectionConfigBuilder(server_config_storage).get()
+        assert config.host == 'myhost.com'
+        assert config.port == 6379
+        assert config.timeout == 0.2
+        assert config.db == RedisConnectionConfigBuilder.DEFAULT_USER_DB
+        assert config.sentinel_id is None
+
+
+class TestRedisMetricsConnectionConfigBuilder(TestCase):
+
+    def test_it_gets_a_valid_config_from_the_server_storage(self):
+        server_config_storage = InMemoryConfigStorage({
+            'redis_metrics_config': {
+                'redis_host': 'myhost.com',
+                'redis_port': 6379,
+                'timeout': 0.2,
+                'redis_db': 3,
+                'sentinel_master_id': 'some_master_id'
+            }
+        })
+        config = RedisMetricsConnectionConfigBuilder(server_config_storage).get()
+        assert config.host == 'myhost.com'
+        assert config.port == 6379
+        assert config.timeout == 0.2
+        assert config.db == 3
+        assert config.sentinel_id == 'some_master_id'

server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py

@@ -0,0 +1,31 @@
+from unittest import TestCase
+from mock import Mock, MagicMock
+from nose.tools import raises
+from cartodb_services.refactor.storage.server_config import *
+import cartodb_services
+
+class TestInDbServerConfigStorage(TestCase):
+
+    def setUp(self):
+        self.plpy_mock = Mock()
+        cartodb_services.init(self.plpy_mock, _GD={})
+
+    def tearDown(self):
+        cartodb_services._reset()
+
+    def test_gets_configs_from_db(self):
+        self.plpy_mock.execute = MagicMock(return_value=[{'conf': '"any_value"'}])
+        server_config = InDbServerConfigStorage()
+        assert server_config.get('any_config') == 'any_value'
+        self.plpy_mock.execute.assert_called_once_with("SELECT cdb_dataservices_server.cdb_conf_getconf('any_config') as conf", 1)
+
+    def test_gets_none_if_cannot_retrieve_key(self):
+        self.plpy_mock.execute = MagicMock(return_value=[{'conf': None}])
+        server_config = InDbServerConfigStorage()
+        assert server_config.get('any_non_existing_key') is None
+
+    def test_deserializes_from_db_to_plain_dict(self):
+        self.plpy_mock.execute = MagicMock(return_value=[{'conf': '{"environment": "testing"}'}])
+        server_config = InDbServerConfigStorage()
+        assert server_config.get('server_conf') == {'environment': 'testing'}
+        self.plpy_mock.execute.assert_called_once_with("SELECT cdb_dataservices_server.cdb_conf_getconf('server_conf') as conf", 1)