diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 7f94d28..7ec2a31 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -210,6 +210,19 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_mapbox_iso_isochrone + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines permission denied + params: + - { name: source, type: "public.geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_tomtom_isochrone return_type: SETOF cdb_dataservices_client.isoline multi_row: true @@ -249,6 +262,19 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_mapbox_iso_isodistance + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + requires_permission: true + permission_name: isolines + permission_error: Isolines permission denied + params: + - { name: source, type: "public.geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_tomtom_isodistance return_type: SETOF cdb_dataservices_client.isoline multi_row: true diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index ade6b65..988855a 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -188,6 +188,70 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_iso_isodistance( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxTrueIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.refactor.service.mapbox_true_isolines_config import MapboxTrueIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxTrueIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + mapbox_iso_isolines = MapboxTrueIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + # -- TODO Support options properly + isolines = {} + for r in data_range: + isoline = mapbox_iso_isolines.calculate_isodistance(origin, r, profile) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations]) + sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox true isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox true isolines') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance( username TEXT, orgname TEXT, @@ -372,6 +436,64 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_iso_isochrones( + username TEXT, + orgname TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxTrueIsolines + from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX + from cartodb_services.tools import Coordinate + from cartodb_services.tools.coordinates import coordinates_to_polygon + from cartodb_services.refactor.service.mapbox_true_isolines_config import MapboxTrueIsolinesConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_manager = ServiceManager('isolines', MapboxTrueIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() + + try: + mapbox_iso_isolines = MapboxTrueIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = Coordinate(lon,lat) + else: + raise Exception('source is NULL') + + profile = TRANSPORT_MODE_TO_MAPBOX.get(mode) + + resp = mapbox_iso_isolines.calculate_isochrone(origin, data_range, profile) + + if resp: + result = [] + for isochrone in resp: + result_polygon = coordinates_to_polygon(isochrone.coordinates) + if result_polygon: + result.append([source, isochrone.duration, result_polygon]) + else: + result.append([source, isochrone.duration, None]) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) + return result + else: + service_manager.quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox true isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get Mapbox true isochrones') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones( username TEXT, orgname TEXT, diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index fbe192e..85811ee 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -26,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_iso_provider: + mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_iso_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) @@ -76,6 +79,20 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return result $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- mapbox true isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_iso_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_iso_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + -- tomtom isodistance CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 731a27e..035ab0f 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -26,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ elif user_isolines_config.mapbox_provider: mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapbox_iso_provider: + mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_iso_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.tomtom_provider: tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options]) @@ -74,6 +77,19 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return result $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +-- mapbox true isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_iso_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_iso_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + -- tomtom isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py index 10ca46e..5709f66 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py @@ -2,4 +2,5 @@ from routing import MapboxRouting, MapboxRoutingResponse from geocoder import MapboxGeocoder from bulk_geocoder import MapboxBulkGeocoder from isolines import MapboxIsolines, MapboxIsochronesResponse +from true_isolines import MapboxTrueIsolines, MapboxTrueIsochronesResponse from matrix_client import MapboxMatrixClient diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py new file mode 100644 index 0000000..2fa53c6 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py @@ -0,0 +1,142 @@ +import json +import requests +from uritemplate import URITemplate + +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools.qps import qps_retry +from cartodb_services.tools import Coordinate + +BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}') + +PROFILE_DRIVING = 'driving' +PROFILE_CYCLING = 'cycling' +PROFILE_WALKING = 'walking' +DEFAULT_PROFILE = PROFILE_DRIVING + +MAX_SPEEDS = { + PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed + PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed + PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed +} + +VALID_PROFILES = [PROFILE_DRIVING, + PROFILE_CYCLING, + PROFILE_WALKING] + +ENTRY_FEATURES = 'features' +ENTRY_GEOMETRY = 'geometry' + + +class MapboxTrueIsolines(): + ''' + Python wrapper for Mapbox based isolines. + ''' + + def __init__(self, apikey, logger, service_params=None): + service_params = service_params or {} + self._apikey = apikey + self._logger = logger + + def _uri(self, origin, time_range, profile=DEFAULT_PROFILE): + uri = URITemplate(BASEURI).expand(apikey=self._apikey, + coordinates=origin, + contours_minutes=time_range, + profile=profile) + return uri + + def _validate_profile(self, profile): + if profile not in VALID_PROFILES: + raise ValueError('{profile} is not a valid profile. ' + 'Valid profiles are: {valid_profiles}'.format( + profile=profile, + valid_profiles=', '.join( + [x for x in VALID_PROFILES]))) + + def _parse_coordinates(self, boundary): + return [Coordinate(c[0], c[1]) for c in boundary] + + def _parse_isochrone_service(self, response): + json_response = json.loads(response) + + coordinates = [] + if json_response: + for feature in json_response[ENTRY_FEATURES]: + geometry = feature[ENTRY_GEOMETRY] + coordinates.append(self._parse_coordinates(geometry)) + + return coordinates + + @qps_retry(qps=5, provider='mapbox_iso') + def _calculate_isoline(self, origin, time_ranges, + profile=DEFAULT_PROFILE): + origin = '{lon},{lat}'.format(lat=origin.latitude, + lon=origin.longitude) + + time_ranges.sort() + time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges]) + + uri = self._uri(origin, time_ranges_seconds, profile) + + try: + response = requests.get(uri) + + if response.status_code == requests.codes.ok: + isolines = [] + + coordinates = self._parse_isochrone_service(response.text) + for t, c in zip(time_ranges, coordinates): + isolines.append(MapboxTrueIsochronesResponse(c, t)) + + return isolines + elif response.status_code == requests.codes.bad_request: + return [] + elif response.status_code == requests.codes.unprocessable_entity: + return [] + else: + raise ServiceException(response.status_code, response) + except requests.Timeout as te: + # In case of timeout we want to stop the job because the server + # could be down + self._logger.error('Timeout connecting to Mapbox isochrone service', + te) + raise ServiceException('Error getting isochrone data from Mapbox', + None) + except requests.ConnectionError as ce: + # Don't raise the exception to continue with the geocoding job + self._logger.error('Error connecting to Mapbox isochrone service', + exception=ce) + return [] + + def calculate_isochrone(self, origin, time_ranges, + profile=DEFAULT_PROFILE): + self._validate_profile(profile) + + return self._calculate_isoline(origin=origin, + time_ranges=time_ranges, + profile=profile) + + def calculate_isodistance(self, origin, distance_range, + profile=DEFAULT_PROFILE): + self._validate_profile(profile) + + max_speed = MAX_SPEEDS[profile] + time_range = distance_range / max_speed + + return self._calculate_isoline(origin=origin, + time_ranges=[time_range], + profile=profile)[0].coordinates + + +class MapboxTrueIsochronesResponse: + + def __init__(self, coordinates, duration): + self._coordinates = coordinates + self._duration = duration + + @property + def coordinates(self): + return self._coordinates + + @property + def duration(self): + return self._duration diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py index cc55bac..57c6764 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py @@ -1,6 +1,7 @@ MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin' MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin' MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin' +MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_iso_isolines_apikey_roundrobin' TRANSPORT_MODE_TO_MAPBOX = { 'car': 'driving', diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 3e45694..6134848 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -224,6 +224,7 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_PROVIDER_KEY = 'geocoder_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' + MAPBOX_ISO_PROVIDER = 'mapbox' TOMTOM_PROVIDER = 'tomtom' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = MAPBOX_PROVIDER @@ -258,6 +259,9 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params + elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER: + self._mapbox_iso_isolines_api_keys = self._db_config.mapbox_iso_isolines_api_keys + self._mapbox_iso_isolines_service_params = db_config.mapbox_iso_isolines_service_params elif self._isolines_provider == self.TOMTOM_PROVIDER: self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params @@ -270,6 +274,8 @@ class IsolinesRoutingConfig(ServiceConfig): return 'mapzen_isolines' elif self._isolines_provider == self.MAPBOX_PROVIDER: return 'mapbox_isolines' + elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER: + return 'mapbox_iso_isolines' elif self._isolines_provider == self.TOMTOM_PROVIDER: return 'tomtom_isolines' @@ -333,6 +339,18 @@ class IsolinesRoutingConfig(ServiceConfig): def mapbox_provider(self): return self._isolines_provider == self.MAPBOX_PROVIDER + @property + def mapbox_iso_isolines_api_keys(self): + return self._mapbox_iso_isolines_api_keys + + @property + def mapbox_iso_isolines_service_params(self): + return self._mapbox_iso_isolines_service_params + + @property + def mapbox_iso_provider(self): + return self._isolines_provider == self.MAPBOX_ISO_PROVIDER + @property def tomtom_isolines_api_keys(self): return self._tomtom_isolines_api_keys @@ -599,6 +617,7 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_mapbox_config() + self._get_mapbox_iso_config() self._get_tomtom_config() self._get_data_observatory_config() @@ -663,6 +682,16 @@ class ServicesDBConfig: self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) + def _get_mapbox_iso_config(self): + mapbox_iso_conf_json = self._get_conf('mapbox_iso_conf') + if not mapbox_iso_conf_json: + raise ConfigException('Mapbox True Isochrones configuration missing') + + mapbox_iso_conf = json.loads(mapbox_iso_conf_json) + self._mapbox_iso_isolines_api_keys = mapbox_conf['isolines']['api_keys'] + self._mapbox_iso_isolines_quota = tomtom_conf['isolines']['monthly_quota'] + self._mapbox_iso_isolines_service_params = tomtom_conf.get('isolines', {}).get('service', {}) + def _get_tomtom_config(self): tomtom_conf_json = self._get_conf('tomtom_conf') if not tomtom_conf_json: @@ -812,6 +841,18 @@ class ServicesDBConfig: def mapbox_geocoder_service_params(self): return self._mapbox_geocoder_service_params + @property + def mapbox_iso_isolines_api_keys(self): + return self._mapbox_iso_isolines_api_keys + + @property + def mapbox_iso_isolines_monthly_quota(self): + return self._mapbox_iso_isolines_quota + + @property + def mapbox_iso_isolines_service_params(self): + return self._mapbox_iso_isolines_service_params + @property def tomtom_isolines_api_keys(self): return self._tomtom_isolines_api_keys diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 70bc9a3..d176183 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -22,6 +22,7 @@ class UserMetricsService: SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' + SERVICE_MAPBOX_ISO_ISOLINES = 'mapbox_iso_isolines' SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_MAPBOX_ROUTING = 'routing_mapbox' @@ -39,6 +40,7 @@ class UserMetricsService: if service_type in [self.SERVICE_HERE_ISOLINES, self.SERVICE_MAPZEN_ISOLINES, self.SERVICE_MAPBOX_ISOLINES, + self.SERVICE_MAPBOX_ISO_ISOLINES, self.SERVICE_TOMTOM_ISOLINES]: return self.__used_isolines_quota(service_type, date) elif service_type in [self.SERVICE_MAPZEN_ROUTING, diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py new file mode 100644 index 0000000..d0864f2 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py @@ -0,0 +1,123 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.mapbox.types import MAPBOX_ISOLINES_APIKEY_ROUNDROBIN + + +class MapboxTrueIsolinesConfig(object): + """ + Configuration needed to operate the Mapbox directions service. + """ + + def __init__(self, + isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + username, + organization, + service_params, + GD): + self._isolines_quota = isolines_quota + self._soft_isolines_limit = soft_isolines_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._mapbox_api_keys = mapbox_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'mapbox_isolines' + + @property + def provider(self): + return 'mapbox' + + @property + def is_high_resolution(self): + return True + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return self._soft_isolines_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def mapbox_api_key(self): + return round_robin(self._mapbox_api_keys, self._GD, + MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class MapboxTrueIsolinesConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + mapbox_server_conf = self._server_conf.get('mapbox_iso_conf') + mapbox_api_keys = mapbox_server_conf['isolines']['api_keys'] + mapbox_service_params = mapbox_server_conf['isolines'].get('service', {}) + + isolines_quota = self._get_quota() + soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('isolines_log_path', None) + + return MapboxTrueIsolinesConfig(isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + self._username, + self._orgname, + mapbox_service_params, + self._GD) + + def _get_quota(self): + isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota') + if isolines_quota is '': + return 0 + + return int(isolines_quota) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 4a272ce..b355eae 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.21.4', + version='0.23.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index bf7d6b5..f2afd1c 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -174,7 +174,7 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): # Don't test mapbox. See CartoDB/cartodb-management/issues/5199" - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom', 'mapbox_iso'] def setUp(self): self.redis_conn = MockRedis() @@ -190,6 +190,8 @@ class TestIsolinesUserConfig(TestCase): assert isolines_config.service_type is 'mapzen_isolines' elif isolines_provider is 'mapbox': assert isolines_config.service_type is 'mapbox_isolines' + elif isolines_provider is 'mapbox_iso': + assert isolines_config.service_type is 'mapbox_iso_isolines' elif isolines_provider is 'tomtom': assert isolines_config.service_type is 'tomtom_isolines' else: diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index dcdbbf5..9052354 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -77,6 +77,7 @@ def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) diff --git a/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py b/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py new file mode 100644 index 0000000..cc9dd9c --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py @@ -0,0 +1,33 @@ +import unittest +from mock import Mock +from cartodb_services.mapbox.true_isolines import MapboxTrueIsolines, DEFAULT_PROFILE +from cartodb_services.tools import Coordinate + +from credentials import mapbox_api_key + +VALID_ORIGIN = Coordinate(-73.989, 40.733) + + +class MapboxTrueIsolinesTestCase(unittest.TestCase): + + def setUp(self): + self.mapbox_isolines = MapboxTrueIsolines(apikey=mapbox_api_key(), + logger=Mock()) + + def test_calculate_isochrone(self): + time_ranges = [300, 900] + solution = self.mapbox_isolines.calculate_isochrone( + origin=VALID_ORIGIN, + profile=DEFAULT_PROFILE, + time_ranges=time_ranges) + + assert solution + + def test_calculate_isodistance(self): + distance_range = 10000 + solution = self.mapbox_isolines.calculate_isodistance( + origin=VALID_ORIGIN, + profile=DEFAULT_PROFILE, + distance_range=distance_range) + + assert solution diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index ce1ca21..9c95dc1 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -178,6 +178,23 @@ class TestQuotaService(TestCase): qs.increment_isolines_service_use(amount=1500000) assert qs.check_user_quota() is False + def test_should_check_user_mapbox_iso_isolines_quota_correctly(self): + qs = self.__build_isolines_quota_service('test_user', + provider='mapbox_iso') + qs.increment_isolines_service_use() + assert qs.check_user_quota() is True + qs.increment_isolines_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_org_mapbox_iso_isolines_quota_correctly(self): + qs = self.__build_isolines_quota_service('test_user', + provider='mapbox_iso', + orgname='testorg') + qs.increment_isolines_service_use() + assert qs.check_user_quota() is True + qs.increment_isolines_service_use(amount=1500000) + assert qs.check_user_quota() is False + # Quick workaround so we don't take into account numer of credits # spent for users that have defined the quota. # See https://github.com/CartoDB/bigmetadata/issues/215