diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 6ad260f..7ba7e1f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -135,7 +135,8 @@ class RoutingConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' - DEFAULT_PROVIDER = 'mapzen' + MAPBOX_PROVIDER = 'mapbox' + DEFAULT_PROVIDER = MAPBOX_PROVIDER QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' METRICS_LOG_KEY = 'routing_log_path' @@ -148,6 +149,8 @@ class RoutingConfig(ServiceConfig): self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key self._mapzen_service_params = self._db_config.mapzen_routing_service_params + self._mapbox_api_key = self._db_config.mapbox_routing_api_key + self._mapbox_service_params = self._db_config.mapbox_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -156,6 +159,8 @@ class RoutingConfig(ServiceConfig): def service_type(self): if self._routing_provider == self.MAPZEN_PROVIDER: return 'routing_mapzen' + elif self._routing_provider == self.MAPBOX_PROVIDER: + return 'routing_mapbox' @property def provider(self): @@ -169,6 +174,14 @@ class RoutingConfig(ServiceConfig): def mapzen_service_params(self): return self._mapzen_service_params + @property + def mapbox_api_key(self): + return self._mapbox_api_key + + @property + def mapbox_service_params(self): + return self._mapbox_service_params + @property def monthly_quota(self): return self._monthly_quota @@ -203,8 +216,9 @@ class IsolinesRoutingConfig(ServiceConfig): ISOLINES_PROVIDER_KEY = 'isolines_provider' GEOCODER_PROVIDER_KEY = 'geocoder_provider' MAPZEN_PROVIDER = 'mapzen' + MAPBOX_PROVIDER = 'mapbox' HEREMAPS_PROVIDER = 'heremaps' - DEFAULT_PROVIDER = 'mapzen' + DEFAULT_PROVIDER = MAPBOX_PROVIDER METRICS_LOG_KEY = 'isolines_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -232,6 +246,10 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params + elif self._isolines_provider == self.MAPBOX_PROVIDER: + self._mapbox_matrix_api_key = self._db_config.mapbox_matrix_api_key + self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params + self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params @property def service_type(self): @@ -239,6 +257,8 @@ class IsolinesRoutingConfig(ServiceConfig): return 'here_isolines' elif self._isolines_provider == self.MAPZEN_PROVIDER: return 'mapzen_isolines' + elif self._isolines_provider == self.MAPBOX_PROVIDER: + return 'mapbox_isolines' @property def google_services_user(self): @@ -284,6 +304,22 @@ class IsolinesRoutingConfig(ServiceConfig): def mapzen_provider(self): return self._isolines_provider == self.MAPZEN_PROVIDER + @property + def mapbox_matrix_api_key(self): + return self._mapbox_matrix_api_key + + @property + def mapbox_matrix_service_params(self): + return self._mapbox_matrix_service_params + + @property + def mapbox_isochrones_service_params(self): + return self._mapbox_isochrones_service_params + + @property + def mapbox_provider(self): + return self._isolines_provider == self.MAPBOX_PROVIDER + @property def heremaps_provider(self): return self._isolines_provider == self.HEREMAPS_PROVIDER @@ -340,12 +376,14 @@ class GeocoderConfig(ServiceConfig): MAPZEN_GEOCODER = 'mapzen' MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key' GEOCODER_PROVIDER = 'geocoder_provider' + MAPBOX_GEOCODER = 'mapbox' + MAPBOX_GEOCODER_API_KEY = 'mapbox_geocoder_api_key' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' - DEFAULT_PROVIDER = 'mapzen' + DEFAULT_PROVIDER = MAPBOX_GEOCODER METRICS_LOG_KEY = 'geocoder_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None): @@ -366,6 +404,9 @@ class GeocoderConfig(ServiceConfig): elif self._geocoder_provider == self.MAPZEN_GEOCODER: if not self.mapzen_api_key: raise ConfigException("""Mapzen config is not set up""") + elif self._geocoder_provider == self.MAPBOX_GEOCODER: + if not self.mapbox_api_key: + raise ConfigException("""Mapbox config is not set up""") return True @@ -397,6 +438,10 @@ class GeocoderConfig(ServiceConfig): self._mapzen_api_key = db_config.mapzen_geocoder_api_key self._cost_per_hit = 0 self._mapzen_service_params = db_config.mapzen_geocoder_service_params + elif self._geocoder_provider == self.MAPBOX_GEOCODER: + self._mapbox_api_key = db_config.mapbox_geocoder_api_key + self._cost_per_hit = 0 + self._mapbox_service_params = db_config.mapbox_geocoder_service_params @property def service_type(self): @@ -404,6 +449,8 @@ class GeocoderConfig(ServiceConfig): return 'geocoder_google' elif self._geocoder_provider == self.MAPZEN_GEOCODER: return 'geocoder_mapzen' + elif self._geocoder_provider == self.MAPBOX_GEOCODER: + return 'geocoder_mapbox' elif self._geocoder_provider == self.NOKIA_GEOCODER: return 'geocoder_here' @@ -419,6 +466,10 @@ class GeocoderConfig(ServiceConfig): def mapzen_geocoder(self): return self._geocoder_provider == self.MAPZEN_GEOCODER + @property + def mapbox_geocoder(self): + return self._geocoder_provider == self.MAPBOX_GEOCODER + @property def google_client_id(self): return self._google_maps_client_id @@ -462,6 +513,14 @@ class GeocoderConfig(ServiceConfig): def mapzen_service_params(self): return self._mapzen_service_params + @property + def mapbox_api_key(self): + return self._mapbox_api_key + + @property + def mapbox_service_params(self): + return self._mapbox_service_params + @property def is_high_resolution(self): return True @@ -478,6 +537,7 @@ class GeocoderConfig(ServiceConfig): def service(self): return self._service + class ServicesDBConfig: def __init__(self, db_conn, username, orgname): @@ -490,6 +550,7 @@ class ServicesDBConfig: self._get_server_config() self._get_here_config() self._get_mapzen_config() + self._get_mapbox_config() self._get_data_observatory_config() def _get_server_config(self): @@ -535,6 +596,23 @@ class ServicesDBConfig: self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) + def _get_mapbox_config(self): + mapbox_conf_json = self._get_conf('mapbox_conf') + if not mapbox_conf_json: + raise ConfigException('Mapzen configuration missing') + else: + mapzen_conf = json.loads(mapbox_conf_json) + self._mapbox_matrix_api_key = mapzen_conf['matrix']['api_key'] + self._mapbox_matrix_quota = mapzen_conf['matrix']['monthly_quota'] + self._mapbox_matrix_service_params = mapzen_conf['matrix'].get('service', {}) + self._mapbox_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {}) + self._mapbox_routing_api_key = mapzen_conf['routing']['api_key'] + self._mapbox_routing_quota = mapzen_conf['routing']['monthly_quota'] + self._mapbox_routing_service_params = mapzen_conf['routing'].get('service', {}) + self._mapbox_geocoder_api_key = mapzen_conf['geocoder']['api_key'] + self._mapbox_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] + self._mapbox_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) + def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') if not do_conf_json: @@ -548,7 +626,6 @@ class ServicesDBConfig: else: self._data_observatory_connection_str = do_conf['connection']['production'] - def _get_conf(self, key): try: sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key) @@ -629,6 +706,46 @@ class ServicesDBConfig: def mapzen_geocoder_service_params(self): return self._mapzen_geocoder_service_params + @property + def mapbox_matrix_api_key(self): + return self._mapbox_matrix_api_key + + @property + def mapbox_matrix_monthly_quota(self): + return self._mapbox_matrix_quota + + @property + def mapbox_matrix_service_params(self): + return self._mapbox_matrix_service_params + + @property + def mapbox_isochrones_service_params(self): + return self._mapbox_isochrones_service_params + + @property + def mapbox_routing_api_key(self): + return self._mapbox_routing_api_key + + @property + def mapbox_routing_monthly_quota(self): + return self._mapbox_routing_quota + + @property + def mapbox_routing_service_params(self): + return self._mapbox_routing_service_params + + @property + def mapbox_geocoder_api_key(self): + return self._mapbox_geocoder_api_key + + @property + def mapbox_geocoder_monthly_quota(self): + return self._mapbox_geocoder_quota + + @property + def mapbox_geocoder_service_params(self): + return self._mapbox_geocoder_service_params + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index b398cdf..1471179 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -72,15 +72,18 @@ class QuotaChecker: if re.match('geocoder_*', self._user_service_config.service_type) is not None: return self.__check_geocoder_quota() - elif re.match('here_isolines', - self._user_service_config.service_type) is not None: - return self.__check_isolines_quota() elif re.match('mapzen_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('mapbox_isolines', + self._user_service_config.service_type) is not None: + return self.__check_isolines_quota() elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() + elif re.match('routing_mapbox', + self._user_service_config.service_type) is not None: + return self.__check_routing_quota() elif re.match('obs_*', self._user_service_config.service_type) is not None: return self.__check_data_observatory_quota() diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index a5f64b6..3c0163e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -21,7 +21,9 @@ class UserMetricsService: SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' + SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' + SERVICE_MAPBOX_ROUTING = 'routing_mapbox' SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' @@ -32,11 +34,12 @@ class UserMetricsService: self._orgname = user_geocoder_config.organization def used_quota(self, service_type, date): - if service_type == self.SERVICE_HERE_ISOLINES: + if service_type in [self.SERVICE_HERE_ISOLINES, + self.SERVICE_MAPZEN_ISOLINES, + self.SERVICE_MAPBOX_ISOLINES]: return self.__used_isolines_quota(service_type, date) - if service_type == self.SERVICE_MAPZEN_ISOLINES: - return self.__used_isolines_quota(service_type, date) - elif service_type == self.SERVICE_MAPZEN_ROUTING: + elif service_type in [self.SERVICE_MAPZEN_ROUTING, + self.SERVICE_MAPBOX_ROUTING]: return self.__used_routing_quota(service_type, date) elif service_type == self.SERVICE_OBSERVATORY: return self.__used_observatory_quota(service_type, date) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/polyline.py b/server/lib/python/cartodb_services/cartodb_services/tools/polyline.py index b3ea775..eaca5f2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/polyline.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/polyline.py @@ -48,7 +48,7 @@ class PolyLine: def polyline_to_linestring(polyline): - """Convert a Mapzen polyline shape to a PostGIS multipolygon""" + """Convert a Mapzen polyline shape to a PostGIS linestring""" coordinates = [] for point in polyline: # Divide by 10 because mapzen uses one more decimal than the diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index b2a184d..89f608e 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -7,7 +7,7 @@ from cartodb_services.metrics.config import * class TestGeocoderUserConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -25,6 +25,9 @@ class TestGeocoderUserConfig(TestCase): elif geocoder_provider == 'mapzen': assert geocoder_config.mapzen_geocoder is True assert geocoder_config.geocoding_quota == 100 + elif geocoder_provider == 'mapbox': + assert geocoder_config.mapbox_geocoder is True + assert geocoder_config.geocoding_quota == 100 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -75,9 +78,10 @@ class TestGeocoderUserConfig(TestCase): 'test_user', None) assert geocoder_config.soft_geocoding_limit == False + class TestGeocoderOrgConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -100,6 +104,9 @@ class TestGeocoderOrgConfig(TestCase): elif geocoder_provider == 'mapzen': assert geocoder_config.mapzen_geocoder is True assert geocoder_config.geocoding_quota == 200 + elif geocoder_provider == 'mapbox': + assert geocoder_config.mapbox_geocoder is True + assert geocoder_config.geocoding_quota == 200 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -160,8 +167,7 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): - - ISOLINES_PROVIDERS = ['heremaps', 'mapzen'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] def setUp(self): self.redis_conn = MockRedis() @@ -175,6 +181,8 @@ class TestIsolinesUserConfig(TestCase): 'test_user') if isolines_provider is 'mapzen': assert isolines_config.service_type is 'mapzen_isolines' + elif isolines_provider is 'mapbox': + assert isolines_config.service_type is 'mapbox_isolines' else: assert isolines_config.service_type is 'here_isolines' assert isolines_config.isolines_quota == 100 @@ -215,9 +223,10 @@ class TestIsolinesUserConfig(TestCase): 'test_user') assert isolines_config.soft_isolines_limit is False + class TestIsolinesOrgConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] def setUp(self): self.redis_conn = MockRedis() diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index fa7f6bd..df0ac67 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -79,6 +79,7 @@ def increment_service_uses(redis_conn, username, orgname=None, def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}]) diff --git a/server/lib/python/cartodb_services/test/test_polyline.py b/server/lib/python/cartodb_services/test/test_polyline.py index 9d5be45..a63ece8 100644 --- a/server/lib/python/cartodb_services/test/test_polyline.py +++ b/server/lib/python/cartodb_services/test/test_polyline.py @@ -35,5 +35,3 @@ class TestPolyline(TestCase): decoded_polyline_3 = self.polyline.decode('ax_~Jv`d~\wrcMa`qj}@dfqaBngtcAhb~Zncc}y@') assert decoded_polyline_3 == original_polyline_3 - - diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 9bd3e4f..f5b47e3 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -97,6 +97,23 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=1500000) assert qs.check_user_quota() is False + def test_should_check_user_mapbox_geocoder_quota_correctly(self): + qs = self.__build_geocoder_quota_service('test_user', + provider='mapbox') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_org_mapbox_geocoder_quota_correctly(self): + qs = self.__build_geocoder_quota_service('test_user', + orgname='testorg', + provider='mapbox') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + def test_should_check_user_routing_quota_correctly(self): qs = self.__build_routing_quota_service('test_user') qs.increment_success_service_use() @@ -179,7 +196,7 @@ class TestQuotaService(TestCase): username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) - def __build_routing_quota_service(self, username, provider='mapzen', + def __build_routing_quota_service(self, username, provider='mapbox', orgname=None, soft_limit=False, quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, 'routing', quota, provider, @@ -188,7 +205,7 @@ class TestQuotaService(TestCase): username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) - def __build_isolines_quota_service(self, username, provider='mapzen', + def __build_isolines_quota_service(self, username, provider='mapbox', orgname=None, soft_limit=False, quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, 'isolines', quota, provider, diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index b45989e..51f72ed 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -97,7 +97,6 @@ class TestUserService(TestCase): super(MockRedisWithCounter, self).__init__() self._zscore_counter = 0 def zscore(self, *args): - print args self._zscore_counter += 1 return super(MockRedisWithCounter, self).zscore(*args) def zscore_counter(self):