Merge pull request #291 from CartoDB/281-quota-mapzen-routing

281 quota mapzen routing

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -116,6 +116,8 @@ class RoutingConfig(ServiceConfig):
     ROUTING_PROVIDER_KEY = 'routing_provider'
     MAPZEN_PROVIDER = 'mapzen'
     DEFAULT_PROVIDER = 'mapzen'
+    QUOTA_KEY = 'mapzen_routing_quota'
+    SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         super(RoutingConfig, self).__init__(redis_connection, db_conn,
@@ -124,7 +126,8 @@ class RoutingConfig(ServiceConfig):
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
         self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
+        self._set_monthly_quota()
+        self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
     @property
@@ -144,6 +147,28 @@ class RoutingConfig(ServiceConfig):
     def period_end_date(self):
         return self._period_end_date
 
+    @property
+    def soft_limit(self):
+        return self._soft_limit
+
+
+    def _set_monthly_quota(self):
+        self._monthly_quota = self._get_effective_monthly_quota()
+
+    def _get_effective_monthly_quota(self):
+        quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
+        if quota_from_redis and quota_from_redis <> '':
+            return int(quota_from_redis)
+        else:
+            return self._db_config.mapzen_routing_monthly_quota
+
+    def _set_soft_limit(self):
+        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
+            self._soft_limit = True
+        else:
+            self._soft_limit = False
+
+
 
 class IsolinesRoutingConfig(ServiceConfig):
 
@@ -547,6 +572,7 @@ class ServicesRedisConfig:
     GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
+    ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
     OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
     OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
@@ -585,8 +611,12 @@ class ServicesRedisConfig:
         if not org_config:
             raise ConfigException("""There is no organization config available. Please check your configuration.'""")
         else:
-            user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
+            if self.QUOTA_KEY in org_config:
-            user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+                user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
+            if self.ISOLINES_QUOTA_KEY in org_config:
+                user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+            if self.ROUTING_QUOTA_KEY in org_config:
+                user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
             if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
                 user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
             if self.OBS_GENERAL_QUOTA_KEY in org_config:

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -93,7 +93,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_geocoding_limit = self._user_service_config.soft_geocoding_limit
 
-        if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -105,7 +105,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_isolines_limit = self._user_service_config.soft_isolines_limit
 
-        if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_isolines_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -115,8 +115,9 @@ class QuotaChecker:
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
+        soft_limit = self._user_service_config.soft_limit
 
-        if (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -128,7 +129,7 @@ class QuotaChecker:
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
 
-        if soft_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -8,6 +8,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
@@ -19,6 +20,8 @@ class UserMetricsService:
     def used_quota(self, service_type, date):
         if service_type == self.SERVICE_HERE_ISOLINES:
             return self.__used_isolines_quota(service_type, date)
+        elif service_type == self.SERVICE_MAPZEN_ROUTING:
+            return self.__used_routing_quota(service_type, date)
         else:
             return self.__used_geocoding_quota(service_type, date)
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
+from ..test_helper import build_plpy_mock
+
+class TestRoutingConfig(TestCase):
+
+    def setUp(self):
+        self._redis_conn = MockRedis()
+        self._db_conn = build_plpy_mock()
+        self._username = 'my_test_user'
+        self._user_key = "rails:users:{0}".format(self._username)
+        self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
+
+    def test_should_pick_quota_from_server_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1500000
+
+    def test_should_pick_quota_from_redis_if_present(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1000
+
+    def test_org_quota_overrides_user_quota(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = 'my_test_org'
+        orgname_key = "rails:orgs:{0}".format(orgname)
+        self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31')
+        self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000)
+
+        # TODO: these are not too relevant for the routing config
+        self._redis_conn.hset(orgname_key, 'geocoding_quota', 0)
+        self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
+
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 5000
+
+
+    def test_should_have_soft_limit_false_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == False
+
+    def test_can_set_soft_limit_in_user_conf(self):
+        self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == True
+
+
+class TestServicesRedisConfig(TestCase):
+    def test_it_picks_mapzen_routing_quota_from_redis(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', None)
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 42
+
+    def test_org_quota_overrides_user_quota(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme')
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 31415

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -0,0 +1,85 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from ..test_helper import build_plpy_mock
+from cartodb_services.metrics.quota import QuotaChecker
+from cartodb_services.metrics import RoutingConfig
+from datetime import datetime
+
+
+class RoutingConfigMock(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestQuotaChecker(TestCase):
+
+    def setUp(self):
+        self.username = 'my_test_user'
+        self.period_end_date = datetime.today()
+        self.service_type = 'routing_mapzen'
+        self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format(
+            self.username,
+            self.service_type,
+            self.period_end_date.year,
+            self.period_end_date.month
+        )
+
+    def test_routing_quota_check_passes_when_enough_quota(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999)
+        assert QuotaChecker(user_service_config, redis_conn).check() == True
+
+    def test_routing_quota_check_fails_when_quota_exhausted(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_fails_right_in_the_limit(self):
+        """
+        I have 1000 credits and I just spent 1000 today. I should not pass
+        the check to perform the 1001th routing operation.
+        """
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = True
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == True

server/lib/python/cartodb_services/test/metrics/test_user.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+from cartodb_services.metrics import UserMetricsService
+import datetime
+from mockredis import MockRedis
+
+class UserGeocoderConfig(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestUserMetricsService(TestCase):
+
+    def setUp(self):
+        user_geocoder_config = UserGeocoderConfig(
+            username = 'my_test_user',
+            organization = None,
+            period_end_date = datetime.date.today()
+        )
+        redis_conn = MockRedis()
+        self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn)
+
+
+    def test_routing_used_quota_zero_when_no_usage(self):
+        assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0
+
+    def test_routing_used_quota_counts_usages(self):
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses')
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses')
+        assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2