From 596189185f3a2f48106eaa1ef2c7311db610cc04 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 2 Mar 2018 16:01:22 +0100 Subject: [PATCH] Added configuration for TomTom services --- README.md | 9 ++ server/extension/test/sql/00_install_test.sql | 1 + .../cartodb_services/metrics/config.py | 111 +++++++++++++++ .../cartodb_services/metrics/quota.py | 6 + .../cartodb_services/metrics/user.py | 8 +- .../service/tomtom_geocoder_config.py | 128 +++++++++++++++++ .../service/tomtom_isolines_config.py | 123 ++++++++++++++++ .../refactor/service/tomtom_routing_config.py | 131 ++++++++++++++++++ .../cartodb_services/tomtom/types.py | 4 + .../cartodb_services/test/credentials.py | 2 + .../test/metrics/test_config.py | 16 ++- .../cartodb_services/test/test_helper.py | 1 + 12 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py diff --git a/README.md b/README.md index e5b0d32..1e9bee7 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,15 @@ SELECT CDB_Conf_SetConf( ); ``` +#### TomTom configuration + +```sql +SELECT CDB_Conf_SetConf( + 'tomtom_conf', + '{"routing": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "isolines": {"api_keys": ["your_api_key"], "monthly_quota": 1500000}}' +); +``` + #### Data Observatory ```sql diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 9ffe2e1..4ce2667 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -14,6 +14,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); +SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index aab4dd5..cf4ac96 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -136,6 +136,7 @@ class RoutingConfig(ServiceConfig): ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' + TOMTOM_PROVIDER = 'tomtom' DEFAULT_PROVIDER = MAPZEN_PROVIDER QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' @@ -151,6 +152,8 @@ class RoutingConfig(ServiceConfig): self._mapzen_service_params = self._db_config.mapzen_routing_service_params self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys self._mapbox_service_params = self._db_config.mapbox_routing_service_params + self._tomtom_api_keys = self._db_config.tomtom_routing_api_keys + self._tomtom_service_params = self._db_config.tomtom_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -161,6 +164,8 @@ class RoutingConfig(ServiceConfig): return 'routing_mapzen' elif self._routing_provider == self.MAPBOX_PROVIDER: return 'routing_mapbox' + elif self._routing_provider == self.TOMTOM_PROVIDER: + return 'routing_tomtom' @property def provider(self): @@ -190,6 +195,18 @@ class RoutingConfig(ServiceConfig): def mapbox_service_params(self): return self._mapbox_service_params + @property + def tomtom_provider(self): + return self._routing_provider == self.TOMTOM_PROVIDER + + @property + def tomtom_api_keys(self): + return self._tomtom_api_keys + + @property + def tomtom_service_params(self): + return self._tomtom_service_params + @property def monthly_quota(self): return self._monthly_quota @@ -225,6 +242,7 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_PROVIDER_KEY = 'geocoder_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' + TOMTOM_PROVIDER = 'tomtom' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = MAPZEN_PROVIDER METRICS_LOG_KEY = 'isolines_log_path' @@ -258,6 +276,9 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params + elif self._isolines_provider == self.TOMTOM_PROVIDER: + self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys + self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params @property def service_type(self): @@ -267,6 +288,8 @@ class IsolinesRoutingConfig(ServiceConfig): return 'mapzen_isolines' elif self._isolines_provider == self.MAPBOX_PROVIDER: return 'mapbox_isolines' + elif self._isolines_provider == self.TOMTOM_PROVIDER: + return 'tomtom_isolines' @property def google_services_user(self): @@ -328,6 +351,18 @@ class IsolinesRoutingConfig(ServiceConfig): def mapbox_provider(self): return self._isolines_provider == self.MAPBOX_PROVIDER + @property + def tomtom_isolines_api_keys(self): + return self._tomtom_isolines_api_keys + + @property + def tomtom_isolines_service_params(self): + return self._tomtom_isolines_service_params + + @property + def tomtom_provider(self): + return self._isolines_provider == self.TOMTOM_PROVIDER + @property def heremaps_provider(self): return self._isolines_provider == self.HEREMAPS_PROVIDER @@ -386,6 +421,8 @@ class GeocoderConfig(ServiceConfig): GEOCODER_PROVIDER = 'geocoder_provider' MAPBOX_GEOCODER = 'mapbox' MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys' + TOMTOM_GEOCODER = 'tomtom' + TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' USERNAME_KEY = 'username' @@ -415,6 +452,9 @@ class GeocoderConfig(ServiceConfig): elif self._geocoder_provider == self.MAPBOX_GEOCODER: if not self.mapbox_api_keys: raise ConfigException("""Mapbox config is not set up""") + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + if not self.tomtom_api_keys: + raise ConfigException("""TomTom config is not set up""") return True @@ -450,6 +490,10 @@ class GeocoderConfig(ServiceConfig): self._mapbox_api_keys = db_config.mapbox_geocoder_api_keys self._cost_per_hit = 0 self._mapbox_service_params = db_config.mapbox_geocoder_service_params + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys + self._cost_per_hit = 0 + self._tomtom_service_params = db_config.tomtom_geocoder_service_params @property def service_type(self): @@ -459,6 +503,8 @@ class GeocoderConfig(ServiceConfig): return 'geocoder_mapzen' elif self._geocoder_provider == self.MAPBOX_GEOCODER: return 'geocoder_mapbox' + elif self._geocoder_provider == self.TOMTOM_GEOCODER: + return 'geocoder_tomtom' elif self._geocoder_provider == self.NOKIA_GEOCODER: return 'geocoder_here' @@ -478,6 +524,10 @@ class GeocoderConfig(ServiceConfig): def mapbox_geocoder(self): return self._geocoder_provider == self.MAPBOX_GEOCODER + @property + def tomtom_geocoder(self): + return self._geocoder_provider == self.TOMTOM_GEOCODER + @property def google_client_id(self): return self._google_maps_client_id @@ -529,6 +579,14 @@ class GeocoderConfig(ServiceConfig): def mapbox_service_params(self): return self._mapbox_service_params + @property + def tomtom_api_keys(self): + return self._tomtom_api_keys + + @property + def tomtom_service_params(self): + return self._tomtom_service_params + @property def is_high_resolution(self): return True @@ -559,6 +617,7 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_mapbox_config() + self._get_tomtom_config() self._get_data_observatory_config() def _get_server_config(self): @@ -621,6 +680,22 @@ class ServicesDBConfig: self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) + def _get_tomtom_config(self): + tomtom_conf_json = self._get_conf('tomtom_conf') + if not tomtom_conf_json: + raise ConfigException('TomTom configuration missing') + else: + tomtom_conf = json.loads(tomtom_conf_json) + self._tomtom_isolines_api_keys = tomtom_conf['isolines']['api_keys'] + self._tomtom_isolines_quota = tomtom_conf['isolines']['monthly_quota'] + self._tomtom_isolines_service_params = tomtom_conf.get('isolines', {}).get('service', {}) + self._tomtom_routing_api_keys = tomtom_conf['routing']['api_keys'] + self._tomtom_routing_quota = tomtom_conf['routing']['monthly_quota'] + self._tomtom_routing_service_params = tomtom_conf['routing'].get('service', {}) + self._tomtom_geocoder_api_keys = tomtom_conf['geocoder']['api_keys'] + self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota'] + self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {}) + def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') if not do_conf_json: @@ -754,6 +829,42 @@ class ServicesDBConfig: def mapbox_geocoder_service_params(self): return self._mapbox_geocoder_service_params + @property + def tomtom_isolines_api_keys(self): + return self._tomtom_isolines_api_keys + + @property + def tomtom_isolines_monthly_quota(self): + return self._tomtom_isolines_quota + + @property + def tomtom_isolines_service_params(self): + return self._tomtom_isolines_service_params + + @property + def tomtom_routing_api_keys(self): + return self._tomtom_routing_api_keys + + @property + def tomtom_routing_monthly_quota(self): + return self._tomtom_routing_quota + + @property + def tomtom_routing_service_params(self): + return self._tomtom_routing_service_params + + @property + def tomtom_geocoder_api_keys(self): + return self._tomtom_geocoder_api_keys + + @property + def tomtom_geocoder_monthly_quota(self): + return self._tomtom_geocoder_quota + + @property + def tomtom_geocoder_service_params(self): + return self._tomtom_geocoder_service_params + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 7be4c6c..62c489a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -81,12 +81,18 @@ class QuotaChecker: elif re.match('mapbox_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('tomtom_isolines', + self._user_service_config.service_type) is not None: + return self.__check_isolines_quota() elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() elif re.match('routing_mapbox', self._user_service_config.service_type) is not None: return self.__check_routing_quota() + elif re.match('routing_tomtom', + self._user_service_config.service_type) is not None: + return self.__check_routing_quota() elif re.match('obs_*', self._user_service_config.service_type) is not None: return self.__check_data_observatory_quota() diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 3c0163e..70bc9a3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -22,8 +22,10 @@ class UserMetricsService: SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' + SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_MAPBOX_ROUTING = 'routing_mapbox' + SERVICE_TOMTOM_ROUTING = 'routing_tomtom' SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' @@ -36,10 +38,12 @@ class UserMetricsService: def used_quota(self, service_type, date): if service_type in [self.SERVICE_HERE_ISOLINES, self.SERVICE_MAPZEN_ISOLINES, - self.SERVICE_MAPBOX_ISOLINES]: + self.SERVICE_MAPBOX_ISOLINES, + self.SERVICE_TOMTOM_ISOLINES]: return self.__used_isolines_quota(service_type, date) elif service_type in [self.SERVICE_MAPZEN_ROUTING, - self.SERVICE_MAPBOX_ROUTING]: + self.SERVICE_MAPBOX_ROUTING, + self.SERVICE_TOMTOM_ROUTING]: return self.__used_routing_quota(service_type, date) elif service_type == self.SERVICE_OBSERVATORY: return self.__used_observatory_quota(service_type, date) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py new file mode 100644 index 0000000..b987e82 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py @@ -0,0 +1,128 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_GEOCODER_APIKEY_ROUNDROBIN + + +class TomTomGeocoderConfig(object): + """ + Configuration needed to operate the TomTom geocoder service. + """ + + def __init__(self, + geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._geocoding_quota = geocoding_quota + self._soft_geocoding_limit = soft_geocoding_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'geocoder_tomtom' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_GEOCODER_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + # TODO: for BW compat, remove + @property + def google_geocoder(self): + return False + + +class TomTomGeocoderConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['geocoder']['api_keys'] + tomtom_service_params = tomtom_server_conf['geocoder'].get('service', {}) + + geocoding_quota = self._get_quota() + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('geocoder_log_path', None) + + return TomTomGeocoderConfig(geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') + if geocoding_quota is '': + return 0 + + return int(geocoding_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py new file mode 100644 index 0000000..1750be1 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py @@ -0,0 +1,123 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_ISOLINES_APIKEY_ROUNDROBIN + + +class TomTomIsolinesConfig(object): + """ + Configuration needed to operate the TomTom directions service. + """ + + def __init__(self, + isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._isolines_quota = isolines_quota + self._soft_isolines_limit = soft_isolines_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'tomtom_isolines' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return self._soft_isolines_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_ISOLINES_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class TomTomIsolinesConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['isolines']['api_keys'] + tomtom_service_params = tomtom_server_conf['isolines'].get('service', {}) + + isolines_quota = self._get_quota() + soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('isolines_log_path', None) + + return TomTomIsolinesConfig(isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota') + if isolines_quota is '': + return 0 + + return int(isolines_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py new file mode 100644 index 0000000..2c9ab9c --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py @@ -0,0 +1,131 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.tomtom.types import TOMTOM_ROUTING_APIKEY_ROUNDROBIN + + +class TomTomRoutingConfig(object): + """ + Configuration needed to operate the TomTom directions service. + """ + + def __init__(self, + routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + username, + organization, + service_params, + GD): + self._routing_quota = routing_quota + self._soft_routing_limit = soft_routing_limit + self._monthly_quota = monthly_quota + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._tomtom_api_keys = tomtom_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'routing_tomtom' + + @property + def provider(self): + return 'tomtom' + + @property + def is_high_resolution(self): + return True + + @property + def routing_quota(self): + return self._routing_quota + + @property + def soft_limit(self): + return self._soft_routing_limit + + @property + def monthly_quota(self): + return self._monthly_quota + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def tomtom_api_key(self): + return round_robin(self._tomtom_api_keys, self._GD, + TOMTOM_ROUTING_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class TomTomRoutingConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + tomtom_server_conf = self._server_conf.get('tomtom_conf') + tomtom_api_keys = tomtom_server_conf['routing']['api_keys'] + monthly_quota = tomtom_server_conf['routing']['monthly_quota'] + tomtom_service_params = tomtom_server_conf['routing'].get('service', {}) + + routing_quota = self._get_quota() + soft_routing_limit = self._user_conf.get('soft_mapzen_routing_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('routing_log_path', None) + + return TomTomRoutingConfig(routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + tomtom_api_keys, + self._username, + self._orgname, + tomtom_service_params, + self._GD) + + def _get_quota(self): + routing_quota = self._org_conf.get('mapzen_routing_quota') or self._user_conf.get('mapzen_routing_quota') + if routing_quota is '': + return 0 + + return int(routing_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py index 63f0b86..dec48c2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/types.py @@ -1,3 +1,7 @@ +TOMTOM_ROUTING_APIKEY_ROUNDROBIN = 'tomtom_routing_apikey_roundrobin' +TOMTOM_GEOCODER_APIKEY_ROUNDROBIN = 'tomtom_geocoder_apikey_roundrobin' +TOMTOM_ISOLINES_APIKEY_ROUNDROBIN = 'tomtom_isolines_apikey_roundrobin' + PROFILE_DRIVING = 'car' PROFILE_CYCLING = 'bicycle' PROFILE_WALKING = 'pedestrian' diff --git a/server/lib/python/cartodb_services/test/credentials.py b/server/lib/python/cartodb_services/test/credentials.py index c818902..52a2a4d 100644 --- a/server/lib/python/cartodb_services/test/credentials.py +++ b/server/lib/python/cartodb_services/test/credentials.py @@ -1,9 +1,11 @@ import os + def mapbox_api_key(): """Returns Mapbox API key. Requires setting MAPBOX_API_KEY environment variable.""" return os.environ['MAPBOX_API_KEY'] + def tomtom_api_key(): """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable.""" return os.environ['TOMTOM_API_KEY'] diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 89f608e..868b963 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -7,7 +7,7 @@ from cartodb_services.metrics.config import * class TestGeocoderUserConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -28,6 +28,9 @@ class TestGeocoderUserConfig(TestCase): elif geocoder_provider == 'mapbox': assert geocoder_config.mapbox_geocoder is True assert geocoder_config.geocoding_quota == 100 + elif geocoder_provider == 'tomtom': + assert geocoder_config.tomtom_geocoder is True + assert geocoder_config.geocoding_quota == 100 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -81,7 +84,7 @@ class TestGeocoderUserConfig(TestCase): class TestGeocoderOrgConfig(TestCase): - GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google'] + GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google'] def setUp(self): self.redis_conn = MockRedis() @@ -107,6 +110,9 @@ class TestGeocoderOrgConfig(TestCase): elif geocoder_provider == 'mapbox': assert geocoder_config.mapbox_geocoder is True assert geocoder_config.geocoding_quota == 200 + elif geocoder_provider == 'tomtom': + assert geocoder_config.tomtom_geocoder is True + assert geocoder_config.geocoding_quota == 200 elif geocoder_provider == 'google': assert geocoder_config.google_geocoder is True assert geocoder_config.geocoding_quota is None @@ -167,7 +173,7 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() @@ -183,6 +189,8 @@ class TestIsolinesUserConfig(TestCase): assert isolines_config.service_type is 'mapzen_isolines' elif isolines_provider is 'mapbox': assert isolines_config.service_type is 'mapbox_isolines' + elif isolines_provider is 'tomtom': + assert isolines_config.service_type is 'tomtom_isolines' else: assert isolines_config.service_type is 'here_isolines' assert isolines_config.isolines_quota == 100 @@ -226,7 +234,7 @@ class TestIsolinesUserConfig(TestCase): class TestIsolinesOrgConfig(TestCase): - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index a8761e1..d8bccea 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -80,6 +80,7 @@ def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])