cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added limits/quotas and tests for Mapbox services

Browse Source
This commit is contained in:
Antonio 2018-01-02 16:21:57 +01:00
parent 15340fedad
commit bb86f080fc
9 changed files with 169 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -135,7 +135,8 @@ class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
METRICS_LOG_KEY = 'routing_log_path'
@ -148,6 +149,8 @@ class RoutingConfig(ServiceConfig):
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
self._mapbox_api_key = self._db_config.mapbox_routing_api_key
self._mapbox_service_params = self._db_config.mapbox_routing_service_params
self._set_monthly_quota()
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@ -156,6 +159,8 @@ class RoutingConfig(ServiceConfig):
def service_type(self):
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
elif self._routing_provider == self.MAPBOX_PROVIDER:
return 'routing_mapbox'
@property
def provider(self):
@ -169,6 +174,14 @@ class RoutingConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def monthly_quota(self):
return self._monthly_quota
@ -203,8 +216,9 @@ class IsolinesRoutingConfig(ServiceConfig):
ISOLINES_PROVIDER_KEY = 'isolines_provider'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
METRICS_LOG_KEY = 'isolines_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
@ -232,6 +246,10 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_PROVIDER:
self._mapbox_matrix_api_key = self._db_config.mapbox_matrix_api_key
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
@property
def service_type(self):
@ -239,6 +257,8 @@ class IsolinesRoutingConfig(ServiceConfig):
return 'here_isolines'
elif self._isolines_provider == self.MAPZEN_PROVIDER:
return 'mapzen_isolines'
elif self._isolines_provider == self.MAPBOX_PROVIDER:
return 'mapbox_isolines'
@property
def google_services_user(self):
@ -284,6 +304,22 @@ class IsolinesRoutingConfig(ServiceConfig):
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_provider(self):
return self._isolines_provider == self.MAPBOX_PROVIDER
@property
def heremaps_provider(self):
return self._isolines_provider == self.HEREMAPS_PROVIDER
@ -340,12 +376,14 @@ class GeocoderConfig(ServiceConfig):
MAPZEN_GEOCODER = 'mapzen'
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
GEOCODER_PROVIDER = 'geocoder_provider'
MAPBOX_GEOCODER = 'mapbox'
MAPBOX_GEOCODER_API_KEY = 'mapbox_geocoder_api_key'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_GEOCODER
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
@ -366,6 +404,9 @@ class GeocoderConfig(ServiceConfig):
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
if not self.mapzen_api_key:
raise ConfigException("""Mapzen config is not set up""")
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
if not self.mapbox_api_key:
raise ConfigException("""Mapbox config is not set up""")
return True
@ -397,6 +438,10 @@ class GeocoderConfig(ServiceConfig):
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._cost_per_hit = 0
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
self._mapbox_api_key = db_config.mapbox_geocoder_api_key
self._cost_per_hit = 0
self._mapbox_service_params = db_config.mapbox_geocoder_service_params
@property
def service_type(self):
@ -404,6 +449,8 @@ class GeocoderConfig(ServiceConfig):
return 'geocoder_google'
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
return 'geocoder_mapzen'
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
return 'geocoder_mapbox'
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
@ -419,6 +466,10 @@ class GeocoderConfig(ServiceConfig):
def mapzen_geocoder(self):
return self._geocoder_provider == self.MAPZEN_GEOCODER
@property
def mapbox_geocoder(self):
return self._geocoder_provider == self.MAPBOX_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@ -462,6 +513,14 @@ class GeocoderConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def is_high_resolution(self):
return True
@ -478,6 +537,7 @@ class GeocoderConfig(ServiceConfig):
def service(self):
return self._service
class ServicesDBConfig:
def __init__(self, db_conn, username, orgname):
@ -490,6 +550,7 @@ class ServicesDBConfig:
self._get_server_config()
self._get_here_config()
self._get_mapzen_config()
self._get_mapbox_config()
self._get_data_observatory_config()
def _get_server_config(self):
@ -535,6 +596,23 @@ class ServicesDBConfig:
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_mapbox_config(self):
mapbox_conf_json = self._get_conf('mapbox_conf')
if not mapbox_conf_json:
raise ConfigException('Mapzen configuration missing')
else:
mapzen_conf = json.loads(mapbox_conf_json)
self._mapbox_matrix_api_key = mapzen_conf['matrix']['api_key']
self._mapbox_matrix_quota = mapzen_conf['matrix']['monthly_quota']
self._mapbox_matrix_service_params = mapzen_conf['matrix'].get('service', {})
self._mapbox_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
self._mapbox_routing_api_key = mapzen_conf['routing']['api_key']
self._mapbox_routing_quota = mapzen_conf['routing']['monthly_quota']
self._mapbox_routing_service_params = mapzen_conf['routing'].get('service', {})
self._mapbox_geocoder_api_key = mapzen_conf['geocoder']['api_key']
self._mapbox_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapbox_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
if not do_conf_json:
@ -548,7 +626,6 @@ class ServicesDBConfig:
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_conf(self, key):
try:
sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
@ -629,6 +706,46 @@ class ServicesDBConfig:
def mapzen_geocoder_service_params(self):
return self._mapzen_geocoder_service_params
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_monthly_quota(self):
return self._mapbox_matrix_quota
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_routing_api_key(self):
return self._mapbox_routing_api_key
@property
def mapbox_routing_monthly_quota(self):
return self._mapbox_routing_quota
@property
def mapbox_routing_service_params(self):
return self._mapbox_routing_service_params
@property
def mapbox_geocoder_api_key(self):
return self._mapbox_geocoder_api_key
@property
def mapbox_geocoder_monthly_quota(self):
return self._mapbox_geocoder_quota
@property
def mapbox_geocoder_service_params(self):
return self._mapbox_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str

9
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -72,15 +72,18 @@ class QuotaChecker:
if re.match('geocoder_*',
self._user_service_config.service_type) is not None:
return self.__check_geocoder_quota()
elif re.match('here_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('mapzen_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('mapbox_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('routing_mapzen',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('routing_mapbox',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('obs_*',
self._user_service_config.service_type) is not None:
return self.__check_data_observatory_quota()

11
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@ -21,7 +21,9 @@ class UserMetricsService:
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
SERVICE_HERE_ISOLINES = 'here_isolines'
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
SERVICE_OBSERVATORY = 'obs_general'
DAY_OF_MONTH_ZERO_PADDED = '%d'
@ -32,11 +34,12 @@ class UserMetricsService:
self._orgname = user_geocoder_config.organization
def used_quota(self, service_type, date):
if service_type == self.SERVICE_HERE_ISOLINES:
if service_type in [self.SERVICE_HERE_ISOLINES,
self.SERVICE_MAPZEN_ISOLINES,
self.SERVICE_MAPBOX_ISOLINES]:
return self.__used_isolines_quota(service_type, date)
if service_type == self.SERVICE_MAPZEN_ISOLINES:
return self.__used_isolines_quota(service_type, date)
elif service_type == self.SERVICE_MAPZEN_ROUTING:
elif service_type in [self.SERVICE_MAPZEN_ROUTING,
self.SERVICE_MAPBOX_ROUTING]:
return self.__used_routing_quota(service_type, date)
elif service_type == self.SERVICE_OBSERVATORY:
return self.__used_observatory_quota(service_type, date)

2
server/lib/python/cartodb_services/cartodb_services/tools/polyline.py
View File

@ -48,7 +48,7 @@ class PolyLine:
def polyline_to_linestring(polyline):
"""Convert a Mapzen polyline shape to a PostGIS multipolygon"""
"""Convert a Mapzen polyline shape to a PostGIS linestring"""
coordinates = []
for point in polyline:
# Divide by 10 because mapzen uses one more decimal than the

19
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
class TestGeocoderUserConfig(TestCase):
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
def setUp(self):
self.redis_conn = MockRedis()
@ -25,6 +25,9 @@ class TestGeocoderUserConfig(TestCase):
elif geocoder_provider == 'mapzen':
assert geocoder_config.mapzen_geocoder is True
assert geocoder_config.geocoding_quota == 100
elif geocoder_provider == 'mapbox':
assert geocoder_config.mapbox_geocoder is True
assert geocoder_config.geocoding_quota == 100
elif geocoder_provider == 'google':
assert geocoder_config.google_geocoder is True
assert geocoder_config.geocoding_quota is None
@ -75,9 +78,10 @@ class TestGeocoderUserConfig(TestCase):
'test_user', None)
assert geocoder_config.soft_geocoding_limit == False
class TestGeocoderOrgConfig(TestCase):
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
def setUp(self):
self.redis_conn = MockRedis()
@ -100,6 +104,9 @@ class TestGeocoderOrgConfig(TestCase):
elif geocoder_provider == 'mapzen':
assert geocoder_config.mapzen_geocoder is True
assert geocoder_config.geocoding_quota == 200
elif geocoder_provider == 'mapbox':
assert geocoder_config.mapbox_geocoder is True
assert geocoder_config.geocoding_quota == 200
elif geocoder_provider == 'google':
assert geocoder_config.google_geocoder is True
assert geocoder_config.geocoding_quota is None
@ -160,8 +167,7 @@ class TestGeocoderOrgConfig(TestCase):
class TestIsolinesUserConfig(TestCase):
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
def setUp(self):
self.redis_conn = MockRedis()
@ -175,6 +181,8 @@ class TestIsolinesUserConfig(TestCase):
'test_user')
if isolines_provider is 'mapzen':
assert isolines_config.service_type is 'mapzen_isolines'
elif isolines_provider is 'mapbox':
assert isolines_config.service_type is 'mapbox_isolines'
else:
assert isolines_config.service_type is 'here_isolines'
assert isolines_config.isolines_quota == 100
@ -215,9 +223,10 @@ class TestIsolinesUserConfig(TestCase):
'test_user')
assert isolines_config.soft_isolines_limit is False
class TestIsolinesOrgConfig(TestCase):
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
def setUp(self):
self.redis_conn = MockRedis()

1
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -79,6 +79,7 @@ def increment_service_uses(redis_conn, username, orgname=None,
def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])

2
server/lib/python/cartodb_services/test/test_polyline.py
View File

@ -35,5 +35,3 @@ class TestPolyline(TestCase):
decoded_polyline_3 = self.polyline.decode('ax_~Jv`d~\wrcMa`qj}@dfqaBngtcAhb~Zncc}y@')
assert decoded_polyline_3 == original_polyline_3

21
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -97,6 +97,23 @@ class TestQuotaService(TestCase):
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_mapbox_geocoder_quota_correctly(self):
qs = self.__build_geocoder_quota_service('test_user',
provider='mapbox')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_org_mapbox_geocoder_quota_correctly(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='testorg',
provider='mapbox')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_routing_quota_correctly(self):
qs = self.__build_routing_quota_service('test_user')
qs.increment_success_service_use()
@ -179,7 +196,7 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
def __build_routing_quota_service(self, username, provider='mapzen',
def __build_routing_quota_service(self, username, provider='mapbox',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, 'routing', quota, provider,
@ -188,7 +205,7 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
def __build_isolines_quota_service(self, username, provider='mapzen',
def __build_isolines_quota_service(self, username, provider='mapbox',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, 'isolines', quota, provider,

1
server/lib/python/cartodb_services/test/test_user_service.py
View File

@ -97,7 +97,6 @@ class TestUserService(TestCase):
super(MockRedisWithCounter, self).__init__()
self._zscore_counter = 0
def zscore(self, *args):
print args
self._zscore_counter += 1
return super(MockRedisWithCounter, self).zscore(*args)
def zscore_counter(self):