cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #291 from CartoDB/281-quota-mapzen-routing

Browse Source 
281 quota mapzen routing
This commit is contained in:
Rafa de la Torre 2016-10-21 17:32:39 +02:00 committed by GitHub
commit e8122c6728
7 changed files with 223 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -116,6 +116,8 @@ class RoutingConfig(ServiceConfig):
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, db_conn,
@ -124,7 +126,8 @@ class RoutingConfig(ServiceConfig):
if not self._routing_provider:
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
self._set_monthly_quota()
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@property
@ -144,6 +147,28 @@ class RoutingConfig(ServiceConfig):
def period_end_date(self):
return self._period_end_date
@property
def soft_limit(self):
return self._soft_limit
def _set_monthly_quota(self):
self._monthly_quota = self._get_effective_monthly_quota()
def _get_effective_monthly_quota(self):
quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
if quota_from_redis and quota_from_redis <> '':
return int(quota_from_redis)
else:
return self._db_config.mapzen_routing_monthly_quota
def _set_soft_limit(self):
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
class IsolinesRoutingConfig(ServiceConfig):
@ -547,6 +572,7 @@ class ServicesRedisConfig:
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
QUOTA_KEY = 'geocoding_quota'
ISOLINES_QUOTA_KEY = 'here_isolines_quota'
ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
@ -585,8 +611,12 @@ class ServicesRedisConfig:
if not org_config:
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
if self.QUOTA_KEY in org_config:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
if self.ISOLINES_QUOTA_KEY in org_config:
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
if self.ROUTING_QUOTA_KEY in org_config:
user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
if self.OBS_GENERAL_QUOTA_KEY in org_config:

9
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -93,7 +93,7 @@ class QuotaChecker:
current_used = self._user_service.used_quota(service_type, today)
soft_geocoding_limit = self._user_service_config.soft_geocoding_limit
if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota):
if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota):
return True
else:
return False
@ -105,7 +105,7 @@ class QuotaChecker:
current_used = self._user_service.used_quota(service_type, today)
soft_isolines_limit = self._user_service_config.soft_isolines_limit
if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota):
if soft_isolines_limit or (user_quota > 0 and current_used < user_quota):
return True
else:
return False
@ -115,8 +115,9 @@ class QuotaChecker:
today = date.today()
service_type = self._user_service_config.service_type
current_used = self._user_service.used_quota(service_type, today)
soft_limit = self._user_service_config.soft_limit
if (user_quota > 0 and current_used <= user_quota):
if soft_limit or (user_quota > 0 and current_used < user_quota):
return True
else:
return False
@ -128,7 +129,7 @@ class QuotaChecker:
service_type = self._user_service_config.service_type
current_used = self._user_service.used_quota(service_type, today)
if soft_limit or (user_quota > 0 and current_used <= user_quota):
if soft_limit or (user_quota > 0 and current_used < user_quota):
return True
else:
return False

3
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@ -8,6 +8,7 @@ class UserMetricsService:
SERVICE_GEOCODER_NOKIA = 'geocoder_here'
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
SERVICE_HERE_ISOLINES = 'here_isolines'
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
DAY_OF_MONTH_ZERO_PADDED = '%d'
def __init__(self, user_geocoder_config, redis_connection):
@ -19,6 +20,8 @@ class UserMetricsService:
def used_quota(self, service_type, date):
if service_type == self.SERVICE_HERE_ISOLINES:
return self.__used_isolines_quota(service_type, date)
elif service_type == self.SERVICE_MAPZEN_ROUTING:
return self.__used_routing_quota(service_type, date)
else:
return self.__used_geocoding_quota(service_type, date)

0
server/lib/python/cartodb_services/test/metrics/__init__.py Normal file
View File

67
server/lib/python/cartodb_services/test/metrics/test_config.py Normal file
View File

@ -0,0 +1,67 @@
from unittest import TestCase
from mockredis import MockRedis
from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
from ..test_helper import build_plpy_mock
class TestRoutingConfig(TestCase):
def setUp(self):
self._redis_conn = MockRedis()
self._db_conn = build_plpy_mock()
self._username = 'my_test_user'
self._user_key = "rails:users:{0}".format(self._username)
self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
def test_should_pick_quota_from_server_by_default(self):
orgname = None
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.monthly_quota == 1500000
def test_should_pick_quota_from_redis_if_present(self):
self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
orgname = None
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.monthly_quota == 1000
def test_org_quota_overrides_user_quota(self):
self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
orgname = 'my_test_org'
orgname_key = "rails:orgs:{0}".format(orgname)
self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31')
self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000)
# TODO: these are not too relevant for the routing config
self._redis_conn.hset(orgname_key, 'geocoding_quota', 0)
self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.monthly_quota == 5000
def test_should_have_soft_limit_false_by_default(self):
orgname = None
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.soft_limit == False
def test_can_set_soft_limit_in_user_conf(self):
self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True)
orgname = None
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.soft_limit == True
class TestServicesRedisConfig(TestCase):
def test_it_picks_mapzen_routing_quota_from_redis(self):
redis_conn = MockRedis()
redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
redis_config = ServicesRedisConfig(redis_conn).build('my_username', None)
assert 'mapzen_routing_quota' in redis_config
assert int(redis_config['mapzen_routing_quota']) == 42
def test_org_quota_overrides_user_quota(self):
redis_conn = MockRedis()
redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415)
redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme')
assert 'mapzen_routing_quota' in redis_config
assert int(redis_config['mapzen_routing_quota']) == 31415

85
server/lib/python/cartodb_services/test/metrics/test_quota.py Normal file
View File

@ -0,0 +1,85 @@
from unittest import TestCase
from mockredis import MockRedis
from ..test_helper import build_plpy_mock
from cartodb_services.metrics.quota import QuotaChecker
from cartodb_services.metrics import RoutingConfig
from datetime import datetime
class RoutingConfigMock(object):
def __init__(self, **kwargs):
self.__dict__ = kwargs
class TestQuotaChecker(TestCase):
def setUp(self):
self.username = 'my_test_user'
self.period_end_date = datetime.today()
self.service_type = 'routing_mapzen'
self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format(
self.username,
self.service_type,
self.period_end_date.year,
self.period_end_date.month
)
def test_routing_quota_check_passes_when_enough_quota(self):
user_service_config = RoutingConfigMock(
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
redis_conn = MockRedis()
redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999)
assert QuotaChecker(user_service_config, redis_conn).check() == True
def test_routing_quota_check_fails_when_quota_exhausted(self):
user_service_config = RoutingConfigMock(
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
redis_conn = MockRedis()
redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
checker = QuotaChecker(user_service_config, redis_conn)
assert checker.check() == False
def test_routing_quota_check_fails_right_in_the_limit(self):
"""
I have 1000 credits and I just spent 1000 today. I should not pass
the check to perform the 1001th routing operation.
"""
user_service_config = RoutingConfigMock(
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
redis_conn = MockRedis()
redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000)
checker = QuotaChecker(user_service_config, redis_conn)
assert checker.check() == False
def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self):
user_service_config = RoutingConfigMock(
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
period_end_date = datetime.today(),
soft_limit = True
)
redis_conn = MockRedis()
redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
checker = QuotaChecker(user_service_config, redis_conn)
assert checker.check() == True

30
server/lib/python/cartodb_services/test/metrics/test_user.py Normal file
View File

@ -0,0 +1,30 @@
from unittest import TestCase
from cartodb_services.metrics import UserMetricsService
import datetime
from mockredis import MockRedis
class UserGeocoderConfig(object):
def __init__(self, **kwargs):
self.__dict__ = kwargs
class TestUserMetricsService(TestCase):
def setUp(self):
user_geocoder_config = UserGeocoderConfig(
username = 'my_test_user',
organization = None,
period_end_date = datetime.date.today()
)
redis_conn = MockRedis()
self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn)
def test_routing_used_quota_zero_when_no_usage(self):
assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0
def test_routing_used_quota_counts_usages(self):
self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses')
self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses')
assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2