Merge pull request #327 from CartoDB/mapzen_services_use_quotas
Mapzen services use quotas
This commit is contained in:
Javier Goizueta
GitHub
commit
ebbfae95cf
@ -55,7 +55,7 @@ class QPSService:
|
||||
|
||||
def retry(self, first_request_time, retry_count):
|
||||
elapsed = datetime.now() - first_request_time
|
||||
if elapsed.microseconds > (self._retry_timeout * 1000.0 * 1000.0):
|
||||
if elapsed.total_seconds() > self._retry_timeout:
|
||||
raise TimeoutException()
|
||||
|
||||
# inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
|
||||
|
||||
@ -43,12 +43,20 @@ class ServiceConfig(object):
|
||||
def metrics_log_path(self):
|
||||
return self._metrics_log_path
|
||||
|
||||
def _get_effective_monthly_quota(self, quota_key, default=0):
|
||||
quota_from_redis = self._redis_config.get(quota_key, None)
|
||||
if quota_from_redis and quota_from_redis <> '':
|
||||
return int(quota_from_redis)
|
||||
else:
|
||||
return default
|
||||
|
||||
def __get_metrics_log_path(self):
|
||||
if self.METRICS_LOG_KEY:
|
||||
return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class DataObservatoryConfig(ServiceConfig):
|
||||
|
||||
METRICS_LOG_KEY = 'do_log_path'
|
||||
@ -92,9 +100,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig):
|
||||
self._soft_limit = True
|
||||
else:
|
||||
self._soft_limit = False
|
||||
self._monthly_quota = 0
|
||||
if self.QUOTA_KEY in self._redis_config:
|
||||
self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
|
||||
self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
|
||||
self._connection_str = self._db_config.data_observatory_connection_str
|
||||
|
||||
@property
|
||||
@ -116,9 +122,7 @@ class ObservatoryConfig(DataObservatoryConfig):
|
||||
self._soft_limit = True
|
||||
else:
|
||||
self._soft_limit = False
|
||||
self._monthly_quota = 0
|
||||
if self.QUOTA_KEY in self._redis_config:
|
||||
self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
|
||||
self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
|
||||
self._connection_str = self._db_config.data_observatory_connection_str
|
||||
|
||||
@property
|
||||
@ -173,14 +177,7 @@ class RoutingConfig(ServiceConfig):
|
||||
return self._soft_limit
|
||||
|
||||
def _set_monthly_quota(self):
|
||||
self._monthly_quota = self._get_effective_monthly_quota()
|
||||
|
||||
def _get_effective_monthly_quota(self):
|
||||
quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
|
||||
if quota_from_redis and quota_from_redis <> '':
|
||||
return int(quota_from_redis)
|
||||
else:
|
||||
return self._db_config.mapzen_routing_monthly_quota
|
||||
self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
|
||||
|
||||
def _set_soft_limit(self):
|
||||
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
||||
@ -217,18 +214,16 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
self._isolines_provider = self.DEFAULT_PROVIDER
|
||||
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
|
||||
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
||||
self._isolines_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
|
||||
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
||||
self._soft_isolines_limit = True
|
||||
else:
|
||||
self._soft_isolines_limit = False
|
||||
if self._isolines_provider == self.HEREMAPS_PROVIDER:
|
||||
self._isolines_quota = int(filtered_config[self.QUOTA_KEY])
|
||||
self._heremaps_app_id = db_config.heremaps_isolines_app_id
|
||||
self._heremaps_app_code = db_config.heremaps_isolines_app_code
|
||||
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
||||
self._soft_isolines_limit = True
|
||||
else:
|
||||
self._soft_isolines_limit = False
|
||||
elif self._isolines_provider == self.MAPZEN_PROVIDER:
|
||||
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
|
||||
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
|
||||
self._soft_isolines_limit = False
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
@ -361,8 +356,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
|
||||
else:
|
||||
self._geocoder_provider = self.DEFAULT_PROVIDER
|
||||
self._geocoding_quota = int(filtered_config[self.QUOTA_KEY])
|
||||
|
||||
self._geocoding_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
|
||||
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
||||
|
||||
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
||||
self._soft_geocoding_limit = True
|
||||
else:
|
||||
@ -377,7 +374,6 @@ class GeocoderConfig(ServiceConfig):
|
||||
self._cost_per_hit = 0
|
||||
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
|
||||
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
|
||||
self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota
|
||||
self._cost_per_hit = 0
|
||||
|
||||
@property
|
||||
|
||||
@ -2,21 +2,25 @@ from datetime import date, timedelta
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from calendar import monthrange
|
||||
|
||||
|
||||
def last_day_of_month(year, month):
|
||||
"""last valid day of a month"""
|
||||
return monthrange(year, month)[1]
|
||||
|
||||
|
||||
def latest_valid_date(year, month, day):
|
||||
"""latest date not later than the day specified"""
|
||||
valid_day = min(day, last_day_of_month(year, month))
|
||||
return date(year, month, valid_day)
|
||||
|
||||
|
||||
class UserMetricsService:
|
||||
""" Class to manage all the user info """
|
||||
|
||||
SERVICE_GEOCODER_NOKIA = 'geocoder_here'
|
||||
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
|
||||
SERVICE_HERE_ISOLINES = 'here_isolines'
|
||||
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
|
||||
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
|
||||
SERVICE_OBSERVATORY = 'obs_general'
|
||||
DAY_OF_MONTH_ZERO_PADDED = '%d'
|
||||
@ -30,6 +34,8 @@ class UserMetricsService:
|
||||
def used_quota(self, service_type, date):
|
||||
if service_type == self.SERVICE_HERE_ISOLINES:
|
||||
return self.__used_isolines_quota(service_type, date)
|
||||
if service_type == self.SERVICE_MAPZEN_ISOLINES:
|
||||
return self.__used_isolines_quota(service_type, date)
|
||||
elif service_type == self.SERVICE_MAPZEN_ROUTING:
|
||||
return self.__used_routing_quota(service_type, date)
|
||||
elif service_type == self.SERVICE_OBSERVATORY:
|
||||
|
||||
@ -67,6 +67,7 @@ class MapzenGeocoderConfig(object):
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._organization
|
||||
@ -86,16 +87,13 @@ class MapzenGeocoderConfigBuilder(object):
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
|
||||
|
||||
def get(self):
|
||||
mapzen_server_conf = self._server_conf.get('mapzen_conf')
|
||||
geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota']
|
||||
mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
|
||||
|
||||
geocoding_quota = self._get_quota(mapzen_server_conf)
|
||||
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
|
||||
|
||||
cost_per_hit=0
|
||||
|
||||
cost_per_hit = 0
|
||||
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
|
||||
period_end_date = date_parse(period_end_date_str)
|
||||
|
||||
@ -110,3 +108,10 @@ class MapzenGeocoderConfigBuilder(object):
|
||||
mapzen_api_key,
|
||||
self._username,
|
||||
self._orgname)
|
||||
|
||||
def _get_quota(self, mapzen_server_conf):
|
||||
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
|
||||
if geocoding_quota is '':
|
||||
return 0
|
||||
|
||||
return int(geocoding_quota)
|
||||
|
||||
@ -10,7 +10,7 @@ from setuptools import setup, find_packages
|
||||
setup(
|
||||
name='cartodb_services',
|
||||
|
||||
version='0.12.4',
|
||||
version='0.13.0',
|
||||
|
||||
description='CartoDB Services API Python Library',
|
||||
|
||||
|
||||
@ -1,7 +1,282 @@
|
||||
from unittest import TestCase
|
||||
from mockredis import MockRedis
|
||||
from datetime import datetime, timedelta
|
||||
from ..test_helper import *
|
||||
from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
|
||||
from cartodb_services.metrics.config import *
|
||||
|
||||
|
||||
class TestGeocoderUserConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_geocoder_config_for_user(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
provider=geocoder_provider, quota=100)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
if geocoder_provider == 'heremaps':
|
||||
assert geocoder_config.heremaps_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'mapzen':
|
||||
assert geocoder_config.mapzen_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
assert geocoder_config.soft_geocoding_limit is False
|
||||
|
||||
def test_should_return_quota_0_when_is_0_in_redis(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota=0, provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
if geocoder_provider is not 'google':
|
||||
assert geocoder_config.geocoding_quota == 0
|
||||
|
||||
def test_should_return_quota_0_if_quota_is_empty(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota='', provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
if geocoder_provider is not 'google':
|
||||
assert geocoder_config.geocoding_quota == 0
|
||||
|
||||
def test_should_return_quota_None_when_is_provider_is_google(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota=0, provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
if geocoder_provider is 'google':
|
||||
assert geocoder_config.geocoding_quota == None
|
||||
|
||||
def test_should_return_true_if_soft_limit_is_true(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota=0, soft_limit=True,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
assert geocoder_config.soft_geocoding_limit == True
|
||||
|
||||
def test_should_return_false_if_soft_limit_is_empty_string(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota=0, soft_limit='',
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
assert geocoder_config.soft_geocoding_limit == False
|
||||
|
||||
class TestGeocoderOrgConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_org_config(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
provider=geocoder_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
|
||||
quota=200, end_date=yesterday,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
if geocoder_provider == 'heremaps':
|
||||
assert geocoder_config.heremaps_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'mapzen':
|
||||
assert geocoder_config.mapzen_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
assert geocoder_config.soft_geocoding_limit is False
|
||||
assert geocoder_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_0_quota_if_has_0_in_redis_config(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
provider=geocoder_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
|
||||
quota=0, end_date=yesterday,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
if geocoder_provider is not 'google':
|
||||
assert geocoder_config.geocoding_quota == 0
|
||||
|
||||
def test_should_return_0_if_quota_is_empty_for_org_in_redis(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
provider=geocoder_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
|
||||
quota='', end_date=yesterday,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
if geocoder_provider is not 'google':
|
||||
assert geocoder_config.geocoding_quota == 0
|
||||
|
||||
def test_should_return_None_if_provider_is_google(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
provider=geocoder_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
|
||||
quota='', end_date=yesterday,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
if geocoder_provider is 'google':
|
||||
assert geocoder_config.geocoding_quota == None
|
||||
|
||||
def test_should_return_user_quota_if_is_not_defined_for_org(self):
|
||||
for geocoder_provider in self.GEOCODER_PROVIDERS:
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
|
||||
quota=100, provider=geocoder_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
|
||||
quota=None, end_date=yesterday,
|
||||
provider=geocoder_provider)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
if geocoder_provider is not 'google':
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
|
||||
|
||||
class TestIsolinesUserConfig(TestCase):
|
||||
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_user_config_for_isolines(self):
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
quota=100, provider=isolines_provider)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
if isolines_provider is 'mapzen':
|
||||
assert isolines_config.service_type is 'mapzen_isolines'
|
||||
else:
|
||||
assert isolines_config.service_type is 'here_isolines'
|
||||
assert isolines_config.isolines_quota == 100
|
||||
assert isolines_config.soft_isolines_limit is False
|
||||
|
||||
def test_should_return_0_quota_for_0_value(self):
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider, quota=0,
|
||||
soft_limit=True)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert isolines_config.isolines_quota == 0
|
||||
|
||||
def test_should_return_0_quota_for_empty_quota_value(self):
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider, quota='')
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert isolines_config.isolines_quota == 0
|
||||
|
||||
def test_should_return_true_soft_limit(self):
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider, quota=0,
|
||||
soft_limit=True)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert isolines_config.soft_isolines_limit is True
|
||||
|
||||
def test_should_return_false_soft_limit_with_empty_string(self):
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider, quota=0,
|
||||
soft_limit='')
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert isolines_config.soft_isolines_limit is False
|
||||
|
||||
class TestIsolinesOrgConfig(TestCase):
|
||||
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_org_config_for_isolines(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
|
||||
quota=200, end_date=yesterday,
|
||||
provider=isolines_provider)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert isolines_config.isolines_quota == 200
|
||||
assert isolines_config.soft_isolines_limit is False
|
||||
assert isolines_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_quota_0_for_0_redis_quota(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider,
|
||||
soft_limit=True)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
|
||||
quota=0, end_date=yesterday,
|
||||
provider=isolines_provider)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert isolines_config.isolines_quota == 0
|
||||
|
||||
def test_should_return_quota_0_for_empty_string_quota_in_org_config(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
|
||||
quota='', end_date=yesterday,
|
||||
provider=isolines_provider)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert isolines_config.isolines_quota == 0
|
||||
|
||||
def test_should_return_user_quota_for_non_existent_org_quota(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
for isolines_provider in self.ISOLINES_PROVIDERS:
|
||||
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
|
||||
provider=isolines_provider, quota=100)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
|
||||
quota=None, end_date=yesterday,
|
||||
provider=isolines_provider)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert isolines_config.isolines_quota == 100
|
||||
|
||||
|
||||
class TestRoutingConfig(TestCase):
|
||||
@ -13,11 +288,6 @@ class TestRoutingConfig(TestCase):
|
||||
self._user_key = "rails:users:{0}".format(self._username)
|
||||
self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
|
||||
|
||||
def test_should_pick_quota_from_server_by_default(self):
|
||||
orgname = None
|
||||
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
|
||||
assert config.monthly_quota == 1500000
|
||||
|
||||
def test_should_pick_quota_from_redis_if_present(self):
|
||||
self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
|
||||
orgname = None
|
||||
@ -38,7 +308,6 @@ class TestRoutingConfig(TestCase):
|
||||
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
|
||||
assert config.monthly_quota == 5000
|
||||
|
||||
|
||||
def test_should_have_soft_limit_false_by_default(self):
|
||||
orgname = None
|
||||
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
|
||||
@ -51,6 +320,136 @@ class TestRoutingConfig(TestCase):
|
||||
assert config.soft_limit == True
|
||||
|
||||
|
||||
class TestDataObservatoryUserConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_config_for_obs_snapshot(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100, end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 100
|
||||
assert do_config.soft_limit is False
|
||||
assert do_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_true_if_soft_limit_is_true_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=0, soft_limit=True, end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.soft_limit is True
|
||||
|
||||
def test_should_return_0_if_quota_is_0_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=0, end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_0_if_quota_is_empty_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota='', end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_config_for_obs_snapshot(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 100
|
||||
assert do_config.soft_limit is False
|
||||
assert do_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_0_if_quota_is_0_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=0, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_0_if_quota_is_empty_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota='', end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_true_if_soft_limit_is_true_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=0, soft_limit=True, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.soft_limit is True
|
||||
|
||||
def test_should_return_true_if_soft_limit_is_empty_string_in_redis(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=0, soft_limit='', end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.soft_limit is False
|
||||
|
||||
class TestDataObservatoryOrgConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_organization_config(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100, end_date=yesterday)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
|
||||
quota=200, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert do_config.monthly_quota == 200
|
||||
assert do_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_quota_0_for_0_in_org_quota_config(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
|
||||
quota=0, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_quota_0_for_empty_in_org_quota_config(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
|
||||
quota='', end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert do_config.monthly_quota == 0
|
||||
|
||||
def test_should_return_user_config_when_org_quota_is_not_defined(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
|
||||
quota=100)
|
||||
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
|
||||
quota=None, end_date=yesterday)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert do_config.monthly_quota == 100
|
||||
|
||||
|
||||
class TestServicesRedisConfig(TestCase):
|
||||
def test_it_picks_mapzen_routing_quota_from_redis(self):
|
||||
redis_conn = MockRedis()
|
||||
|
||||
@ -0,0 +1,154 @@
|
||||
from unittest import TestCase
|
||||
from mockredis import MockRedis
|
||||
from datetime import datetime
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import *
|
||||
from cartodb_services.refactor.storage.redis_config import *
|
||||
from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
|
||||
|
||||
|
||||
class TestMapzenGeocoderUserConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._redis_connection = MockRedis()
|
||||
self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
|
||||
"mapzen_conf":
|
||||
{"geocoder":
|
||||
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
|
||||
}, "logger_conf": {}})
|
||||
self._username = 'test_user'
|
||||
self._user_key = "rails:users:{0}".format(self._username)
|
||||
self._user_config = RedisUserConfigStorageBuilder(self._redis_connection,
|
||||
self._username).get()
|
||||
self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection,
|
||||
None).get()
|
||||
self._set_default_config_values()
|
||||
|
||||
def test_config_values_are_ok(self):
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.geocoding_quota == 100
|
||||
assert config.soft_geocoding_limit == False
|
||||
assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S")
|
||||
assert config.service_type == 'geocoder_mapzen'
|
||||
assert config.provider == 'mapzen'
|
||||
assert config.is_high_resolution == True
|
||||
assert config.cost_per_hit == 0
|
||||
assert config.mapzen_api_key == 'search-xxxxxxx'
|
||||
assert config.username == 'test_user'
|
||||
assert config.organization is None
|
||||
|
||||
def test_quota_should_be_0_if_redis_value_is_0(self):
|
||||
self._redis_connection.hset(self._user_key, 'geocoding_quota', '0')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.geocoding_quota == 0
|
||||
|
||||
def test_quota_should_be_0_if_redis_value_is_empty_string(self):
|
||||
self._redis_connection.hset(self._user_key, 'geocoding_quota', '')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.geocoding_quota == 0
|
||||
|
||||
def test_soft_limit_should_be_true(self):
|
||||
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'true')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.soft_geocoding_limit == True
|
||||
|
||||
def test_soft_limit_should_be_false_if_is_empty_string(self):
|
||||
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', '')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.soft_geocoding_limit == False
|
||||
|
||||
def _set_default_config_values(self):
|
||||
self._redis_connection.hset(self._user_key, 'geocoding_quota', '100')
|
||||
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
|
||||
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00')
|
||||
|
||||
class TestMapzenGeocoderOrgConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._redis_connection = MockRedis()
|
||||
self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
|
||||
"mapzen_conf":
|
||||
{"geocoder":
|
||||
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
|
||||
}, "logger_conf": {}})
|
||||
self._username = 'test_user'
|
||||
self._organization = 'test_org'
|
||||
self._user_key = "rails:users:{0}".format(self._username)
|
||||
self._org_key = "rails:orgs:{0}".format(self._organization)
|
||||
self._user_config = RedisUserConfigStorageBuilder(self._redis_connection,
|
||||
self._username).get()
|
||||
self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection,
|
||||
self._organization).get()
|
||||
self._set_default_config_values()
|
||||
|
||||
|
||||
def test_config_org_values_are_ok(self):
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
self._organization).get()
|
||||
assert config.geocoding_quota == 200
|
||||
assert config.soft_geocoding_limit == False
|
||||
assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S")
|
||||
assert config.service_type == 'geocoder_mapzen'
|
||||
assert config.provider == 'mapzen'
|
||||
assert config.is_high_resolution == True
|
||||
assert config.cost_per_hit == 0
|
||||
assert config.mapzen_api_key == 'search-xxxxxxx'
|
||||
assert config.username == 'test_user'
|
||||
assert config.organization is 'test_org'
|
||||
|
||||
def test_quota_should_be_0_if_redis_value_is_0(self):
|
||||
self._redis_connection.hset(self._org_key, 'geocoding_quota', '0')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
self._organization).get()
|
||||
assert config.geocoding_quota == 0
|
||||
|
||||
def test_quota_should_use_user_quota_value_if_redis_value_is_empty_string(self):
|
||||
self._redis_connection.hset(self._org_key, 'geocoding_quota', '')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
self._organization).get()
|
||||
assert config.geocoding_quota == 100
|
||||
|
||||
def test_quota_should_be_0_if_both_user_and_org_have_empty_string(self):
|
||||
self._redis_connection.hset(self._user_key, 'geocoding_quota', '')
|
||||
self._redis_connection.hset(self._org_key, 'geocoding_quota', '')
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
self._organization).get()
|
||||
assert config.geocoding_quota == 0
|
||||
|
||||
def _set_default_config_values(self):
|
||||
self._redis_connection.hset(self._user_key, 'geocoding_quota', '100')
|
||||
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
|
||||
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00')
|
||||
self._redis_connection.hset(self._org_key, 'geocoding_quota', '200')
|
||||
self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00')
|
||||
@ -1,60 +0,0 @@
|
||||
from test_helper import *
|
||||
from unittest import TestCase
|
||||
from nose.tools import assert_raises
|
||||
from mockredis import MockRedis
|
||||
from datetime import datetime, timedelta
|
||||
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
|
||||
|
||||
|
||||
class TestConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
plpy_mock_config()
|
||||
|
||||
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
|
||||
build_redis_user_config(self.redis_conn, 'test_user')
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', None)
|
||||
assert geocoder_config.heremaps_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
assert geocoder_config.soft_geocoding_limit is False
|
||||
|
||||
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user')
|
||||
build_redis_org_config(self.redis_conn, 'test_org',
|
||||
quota=200, end_date=yesterday)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
'test_user', 'test_org')
|
||||
assert geocoder_config.heremaps_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
assert geocoder_config.soft_geocoding_limit is False
|
||||
assert geocoder_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_config_for_obs_snapshot(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user',
|
||||
do_quota=100, soft_do_limit=True,
|
||||
end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 100
|
||||
assert do_config.soft_limit is True
|
||||
assert do_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
|
||||
yesterday = datetime.today() - timedelta(days=1)
|
||||
build_redis_user_config(self.redis_conn, 'test_user',
|
||||
end_date=yesterday)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
'test_user')
|
||||
assert do_config.monthly_quota == 0
|
||||
assert do_config.soft_limit is False
|
||||
assert do_config.period_end_date.date() == yesterday.date()
|
||||
|
||||
def test_should_raise_exception_when_missing_parameters(self):
|
||||
plpy_mock._reset()
|
||||
build_redis_user_config(self.redis_conn, 'test_user')
|
||||
assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
|
||||
plpy_mock, 'test_user', None)
|
||||
@ -1,4 +1,5 @@
|
||||
from datetime import datetime, date
|
||||
from dateutil.tz import tzlocal
|
||||
from mock import Mock, MagicMock
|
||||
import random
|
||||
import sys
|
||||
@ -8,46 +9,60 @@ plpy_mock = MockPlPy()
|
||||
sys.modules['plpy'] = plpy_mock
|
||||
|
||||
|
||||
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
|
||||
service="heremaps", isolines_quota=0,
|
||||
do_quota=None, soft_do_limit=None,
|
||||
do_general_quota=None, soft_do_general_limit=None,
|
||||
def build_redis_user_config(redis_conn, username, service, quota=100,
|
||||
soft_limit=False, provider="heremaps",
|
||||
end_date=datetime.today()):
|
||||
end_date_tz = end_date.replace(tzinfo=tzlocal())
|
||||
user_redis_name = "rails:users:{0}".format(username)
|
||||
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
|
||||
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
|
||||
redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota)
|
||||
redis_conn.hset(user_redis_name, 'geocoder_provider', service)
|
||||
redis_conn.hset(user_redis_name, 'isolines_provider', service)
|
||||
redis_conn.hset(user_redis_name, 'routing_provider', service)
|
||||
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
|
||||
if do_quota:
|
||||
redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota)
|
||||
if soft_do_limit:
|
||||
redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit',
|
||||
soft_do_limit)
|
||||
if do_general_quota:
|
||||
redis_conn.hset(user_redis_name, 'obs_general_quota', do_general_quota)
|
||||
if soft_do_general_limit:
|
||||
redis_conn.hset(user_redis_name, 'soft_obs_general_limit',
|
||||
soft_do_general_limit)
|
||||
|
||||
if service is 'geocoding':
|
||||
redis_conn.hset(user_redis_name, 'geocoder_provider', provider)
|
||||
redis_conn.hset(user_redis_name, 'geocoding_quota', str(quota))
|
||||
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', str(soft_limit).lower())
|
||||
elif service is 'isolines':
|
||||
redis_conn.hset(user_redis_name, 'isolines_provider', provider)
|
||||
redis_conn.hset(user_redis_name, 'here_isolines_quota', str(quota))
|
||||
redis_conn.hset(user_redis_name, 'soft_here_isolines_limit', str(soft_limit).lower())
|
||||
elif service is 'routing':
|
||||
redis_conn.hset(user_redis_name, 'routing_provider', provider)
|
||||
redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota))
|
||||
redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower())
|
||||
elif service is 'data_observatory':
|
||||
redis_conn.hset(user_redis_name, 'obs_snapshot_quota', str(quota))
|
||||
redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota))
|
||||
redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', str(soft_limit).lower())
|
||||
redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower())
|
||||
|
||||
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
|
||||
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
|
||||
redis_conn.hset(user_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z"))
|
||||
|
||||
|
||||
def build_redis_org_config(redis_conn, orgname, quota=100, service="heremaps",
|
||||
isolines_quota=0, do_quota=None,
|
||||
do_general_quota=None, end_date=datetime.today()):
|
||||
def build_redis_org_config(redis_conn, orgname, service, quota=100,
|
||||
provider="heremaps", end_date=datetime.now(tzlocal())):
|
||||
org_redis_name = "rails:orgs:{0}".format(orgname)
|
||||
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
|
||||
redis_conn.hset(org_redis_name, 'here_isolines_quota', isolines_quota)
|
||||
if do_quota:
|
||||
redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota)
|
||||
if do_general_quota:
|
||||
redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota)
|
||||
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
|
||||
end_date_tz = end_date.replace(tzinfo=tzlocal())
|
||||
|
||||
if service is 'geocoding':
|
||||
redis_conn.hset(org_redis_name, 'geocoder_provider', provider)
|
||||
if quota is not None:
|
||||
redis_conn.hset(org_redis_name, 'geocoding_quota', str(quota))
|
||||
elif service is 'isolines':
|
||||
redis_conn.hset(org_redis_name, 'isolines_provider', provider)
|
||||
if quota is not None:
|
||||
redis_conn.hset(org_redis_name, 'here_isolines_quota', str(quota))
|
||||
elif service is 'routing':
|
||||
redis_conn.hset(org_redis_name, 'routing_provider', provider)
|
||||
if quota is not None:
|
||||
redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota))
|
||||
elif service is 'data_observatory':
|
||||
if quota is not None:
|
||||
redis_conn.hset(org_redis_name, 'obs_snapshot_quota', str(quota))
|
||||
redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota))
|
||||
|
||||
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
|
||||
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
|
||||
redis_conn.hset(org_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z"))
|
||||
|
||||
|
||||
def increment_service_uses(redis_conn, username, orgname=None,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
from test_helper import *
|
||||
from mockredis import MockRedis
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
|
||||
from cartodb_services.metrics import *
|
||||
from unittest import TestCase
|
||||
from nose.tools import assert_raises
|
||||
from datetime import datetime, date
|
||||
@ -35,26 +35,26 @@ class TestQuotaService(TestCase):
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
orgname='test_org')
|
||||
increment_service_uses(self.redis_conn, 'test_user',
|
||||
orgname='test_org')
|
||||
orgname='test_org')
|
||||
assert qs.check_user_quota() is True
|
||||
|
||||
def test_should_return_false_if_user_quota_is_surpassed(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user')
|
||||
increment_service_uses(self.redis_conn, 'test_user',
|
||||
amount=300)
|
||||
amount=300)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_return_false_if_org_quota_is_surpassed(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
orgname='test_org')
|
||||
increment_service_uses(self.redis_conn, 'test_user',
|
||||
orgname='test_org', amount=400)
|
||||
orgname='test_org', amount=400)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
|
||||
increment_service_uses(self.redis_conn, 'test_user',
|
||||
amount=300)
|
||||
amount=300)
|
||||
assert qs.check_user_quota() is True
|
||||
|
||||
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
|
||||
@ -62,7 +62,7 @@ class TestQuotaService(TestCase):
|
||||
orgname='test_org',
|
||||
soft_limit=True)
|
||||
increment_service_uses(self.redis_conn, 'test_user',
|
||||
orgname='test_org', amount=400)
|
||||
orgname='test_org', amount=400)
|
||||
assert qs.check_user_quota() is True
|
||||
|
||||
def test_should_check_user_increment_and_quota_check_correctly(self):
|
||||
@ -81,15 +81,17 @@ class TestQuotaService(TestCase):
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_mapzen_geocoder_quota_correctly(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user', service='mapzen')
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
provider='mapzen')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_mapzen_geocoder_quota_correctly(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user', orgname='testorg',
|
||||
service='mapzen')
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
orgname='testorg',
|
||||
provider='mapzen')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
@ -111,16 +113,17 @@ class TestQuotaService(TestCase):
|
||||
|
||||
def test_should_check_user_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user')
|
||||
qs.increment_success_service_use()
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
qs.increment_isolines_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user', orgname='testorg')
|
||||
qs.increment_success_service_use()
|
||||
qs = self.__build_isolines_quota_service('test_user',
|
||||
orgname='testorg')
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
qs.increment_isolines_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_obs_snapshot_quota_correctly(self):
|
||||
@ -138,55 +141,77 @@ class TestQuotaService(TestCase):
|
||||
qs.increment_success_service_use(amount=100000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def __prepare_quota_service(self, username, quota, service, orgname,
|
||||
soft_limit, do_quota, soft_do_limit, end_date):
|
||||
build_redis_user_config(self.redis_conn, username,
|
||||
quota=quota, service=service,
|
||||
soft_limit=soft_limit,
|
||||
soft_do_limit=soft_do_limit,
|
||||
do_quota=do_quota,
|
||||
end_date=end_date)
|
||||
def test_should_check_user_obs_quota_correctly(self):
|
||||
qs = self.__build_obs_snapshot_quota_service('test_user')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=100000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_obs_quota_correctly(self):
|
||||
qs = self.__build_obs_quota_service('test_user',
|
||||
orgname='testorg')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=100000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def __prepare_quota_service(self, username, service, quota, provider,
|
||||
orgname, soft_limit, end_date):
|
||||
build_redis_user_config(self.redis_conn, username, service,
|
||||
quota=quota, provider=provider,
|
||||
soft_limit=soft_limit, end_date=end_date)
|
||||
if orgname:
|
||||
build_redis_org_config(self.redis_conn, orgname,
|
||||
quota=quota, service=service,
|
||||
do_quota=do_quota,
|
||||
end_date=end_date)
|
||||
build_redis_org_config(self.redis_conn, orgname, service,
|
||||
quota=quota, provider=provider,
|
||||
end_date=end_date)
|
||||
|
||||
def __build_geocoder_quota_service(self, username, quota=100,
|
||||
service='heremaps', orgname=None,
|
||||
provider='heremaps', orgname=None,
|
||||
soft_limit=False,
|
||||
end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, quota, service, orgname,
|
||||
soft_limit, 0, False, end_date)
|
||||
self.__prepare_quota_service(username, 'geocoding', quota,
|
||||
provider, orgname, soft_limit, end_date)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_routing_quota_service(self, username, service='mapzen',
|
||||
def __build_routing_quota_service(self, username, provider='mapzen',
|
||||
orgname=None, soft_limit=False,
|
||||
quota=100, end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, quota, service, orgname,
|
||||
soft_limit, 0, False, end_date)
|
||||
self.__prepare_quota_service(username, 'routing', quota, provider,
|
||||
orgname, soft_limit, end_date)
|
||||
routing_config = RoutingConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return QuotaService(routing_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_isolines_quota_service(self, username, service='mapzen',
|
||||
def __build_isolines_quota_service(self, username, provider='mapzen',
|
||||
orgname=None, soft_limit=False,
|
||||
quota=100, end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, quota, service, orgname,
|
||||
soft_limit, 0, False, end_date)
|
||||
self.__prepare_quota_service(username, 'isolines', quota, provider,
|
||||
orgname, soft_limit, end_date)
|
||||
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return QuotaService(isolines_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_obs_snapshot_quota_service(self, username, quota=100,
|
||||
service='obs_snapshot',
|
||||
orgname=None,
|
||||
soft_limit=False,
|
||||
end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, 0, service, orgname, False,
|
||||
quota, soft_limit, end_date)
|
||||
provider='obs_snapshot',
|
||||
orgname=None,
|
||||
soft_limit=False,
|
||||
end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, 'data_observatory', quota,
|
||||
None, orgname, soft_limit, end_date)
|
||||
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return QuotaService(do_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_obs_quota_service(self, username, quota=100,
|
||||
provider='obs_general',
|
||||
orgname=None,
|
||||
soft_limit=False,
|
||||
end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, 'data_observatory', quota,
|
||||
None, orgname, soft_limit, end_date)
|
||||
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return QuotaService(do_config, redis_connection=self.redis_conn)
|
||||
|
||||
@ -24,7 +24,7 @@ class TestUserService(TestCase):
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
|
||||
|
||||
def test_user_quota_for_a_month_shorter_than_end_day(self):
|
||||
us = self.__build_user_service('test_user', end_date=date(2016,1,31))
|
||||
us = self.__build_user_service('test_user', end_date=datetime(2016,1,31))
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
|
||||
|
||||
def test_org_used_quota_for_a_day(self):
|
||||
@ -35,7 +35,7 @@ class TestUserService(TestCase):
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
|
||||
|
||||
def test_org_quota_quota_for_a_month_shorter_than_end_day(self):
|
||||
us = self.__build_user_service('test_user', orgname='test_org', end_date=date(2016,1,31))
|
||||
us = self.__build_user_service('test_user', orgname='test_org', end_date=datetime(2016,1,31))
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
|
||||
|
||||
def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
|
||||
@ -116,7 +116,7 @@ class TestUserService(TestCase):
|
||||
def zscore_counter(self):
|
||||
return self._zscore_counter
|
||||
self.redis_conn = MockRedisWithCounter()
|
||||
us = self.__build_user_service('test_user', end_date=date.today())
|
||||
us = self.__build_user_service('test_user', end_date=datetime.today())
|
||||
us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15))
|
||||
|
||||
#('user:test_user:geocoder_here:success_responses:201506', 15)
|
||||
@ -140,16 +140,17 @@ class TestUserService(TestCase):
|
||||
assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506', '1') == None
|
||||
|
||||
|
||||
def __build_user_service(self, username, quota=100, service='heremaps',
|
||||
orgname=None, soft_limit=False,
|
||||
end_date=date.today()):
|
||||
build_redis_user_config(self.redis_conn, username,
|
||||
quota=quota, service=service,
|
||||
soft_limit=soft_limit,
|
||||
end_date=end_date)
|
||||
def __build_user_service(self, username, service='geocoding', quota=100,
|
||||
provider='heremaps', orgname=None,
|
||||
soft_limit=False, end_date=datetime.today()):
|
||||
build_redis_user_config(self.redis_conn, username, service,
|
||||
quota=quota, provider=provider,
|
||||
soft_limit=soft_limit,
|
||||
end_date=end_date)
|
||||
if orgname:
|
||||
build_redis_org_config(self.redis_conn, orgname,
|
||||
quota=quota, end_date=end_date)
|
||||
build_redis_org_config(self.redis_conn, orgname, service,
|
||||
provider=provider, quota=quota,
|
||||
end_date=end_date)
|
||||
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
|
||||
username, orgname)
|
||||
return UserMetricsService(geocoder_config, self.redis_conn)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user