cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

First implementation of the true Mapbox isochrones

Browse Source
This commit is contained in:
antoniocarlon 2020-01-17 17:04:43 +01:00
parent d2be601049
commit 2a68291da7
15 changed files with 546 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
client/renderer/interface.yaml
View File

@ -210,6 +210,19 @@
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_mapbox_iso_isochrone
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true
multi_field: true
requires_permission: true
permission_name: isolines
permission_error: Isolines permission denied
params:
- { name: source, type: "public.geometry(Geometry, 4326)" }
- { name: mode, type: text }
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_tomtom_isochrone
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true
@ -249,6 +262,19 @@
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_mapbox_iso_isodistance
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true
multi_field: true
requires_permission: true
permission_name: isolines
permission_error: Isolines permission denied
params:
- { name: source, type: "public.geometry(Geometry, 4326)" }
- { name: mode, type: text }
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_tomtom_isodistance
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true

122
server/extension/sql/80_isolines_helper.sql
View File

@ -188,6 +188,70 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_iso_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxTrueIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_true_isolines_config import MapboxTrueIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxTrueIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
mapbox_iso_isolines = MapboxTrueIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_iso_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox true isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox true isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
username TEXT,
orgname TEXT,
@ -372,6 +436,64 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_iso_isochrones(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxTrueIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
from cartodb_services.refactor.service.mapbox_true_isolines_config import MapboxTrueIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxTrueIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
mapbox_iso_isolines = MapboxTrueIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
resp = mapbox_iso_isolines.calculate_isochrone(origin, data_range, profile)
if resp:
result = []
for isochrone in resp:
result_polygon = coordinates_to_polygon(isochrone.coordinates)
if result_polygon:
result.append([source, isochrone.duration, result_polygon])
else:
result.append([source, isochrone.duration, None])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(result))
return result
else:
service_manager.quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox true isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox true isochrones')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
username TEXT,
orgname TEXT,

17
server/extension/sql/85_isodistance.sql
View File

@ -26,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
elif user_isolines_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapbox_iso_provider:
mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_iso_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
@ -76,6 +79,20 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
-- mapbox true isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_iso_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_iso_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
-- tomtom isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$

16
server/extension/sql/90_isochrone.sql
View File

@ -26,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
elif user_isolines_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapbox_iso_provider:
mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_iso_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
@ -74,6 +77,19 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
-- mapbox true isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_iso_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
mapbox_iso_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_iso_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapbox_iso_plan, [username, orgname, source, mode, range, options])
return result
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
-- tomtom isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$

1
server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py
View File

@ -2,4 +2,5 @@ from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder
from bulk_geocoder import MapboxBulkGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse
from true_isolines import MapboxTrueIsolines, MapboxTrueIsochronesResponse
from matrix_client import MapboxMatrixClient

142
server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py Normal file
View File

@ -0,0 +1,142 @@
import json
import requests
from uritemplate import URITemplate
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate
BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
PROFILE_DRIVING = 'driving'
PROFILE_CYCLING = 'cycling'
PROFILE_WALKING = 'walking'
DEFAULT_PROFILE = PROFILE_DRIVING
MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed
PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed
}
VALID_PROFILES = [PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING]
ENTRY_FEATURES = 'features'
ENTRY_GEOMETRY = 'geometry'
class MapboxTrueIsolines():
'''
Python wrapper for Mapbox based isolines.
'''
def __init__(self, apikey, logger, service_params=None):
service_params = service_params or {}
self._apikey = apikey
self._logger = logger
def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
uri = URITemplate(BASEURI).expand(apikey=self._apikey,
coordinates=origin,
contours_minutes=time_range,
profile=profile)
return uri
def _validate_profile(self, profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
def _parse_coordinates(self, boundary):
return [Coordinate(c[0], c[1]) for c in boundary]
def _parse_isochrone_service(self, response):
json_response = json.loads(response)
coordinates = []
if json_response:
for feature in json_response[ENTRY_FEATURES]:
geometry = feature[ENTRY_GEOMETRY]
coordinates.append(self._parse_coordinates(geometry))
return coordinates
@qps_retry(qps=5, provider='mapbox_iso')
def _calculate_isoline(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
origin = '{lon},{lat}'.format(lat=origin.latitude,
lon=origin.longitude)
time_ranges.sort()
time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
uri = self._uri(origin, time_ranges_seconds, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
isolines = []
coordinates = self._parse_isochrone_service(response.text)
for t, c in zip(time_ranges, coordinates):
isolines.append(MapboxTrueIsochronesResponse(c, t))
return isolines
elif response.status_code == requests.codes.bad_request:
return []
elif response.status_code == requests.codes.unprocessable_entity:
return []
else:
raise ServiceException(response.status_code, response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapbox isochrone service',
te)
raise ServiceException('Error getting isochrone data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox isochrone service',
exception=ce)
return []
def calculate_isochrone(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
self._validate_profile(profile)
return self._calculate_isoline(origin=origin,
time_ranges=time_ranges,
profile=profile)
def calculate_isodistance(self, origin, distance_range,
profile=DEFAULT_PROFILE):
self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile]
time_range = distance_range / max_speed
return self._calculate_isoline(origin=origin,
time_ranges=[time_range],
profile=profile)[0].coordinates
class MapboxTrueIsochronesResponse:
def __init__(self, coordinates, duration):
self._coordinates = coordinates
self._duration = duration
@property
def coordinates(self):
return self._coordinates
@property
def duration(self):
return self._duration

1
server/lib/python/cartodb_services/cartodb_services/mapbox/types.py
View File

@ -1,6 +1,7 @@
MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin'
MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin'
MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin'
MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_iso_isolines_apikey_roundrobin'
TRANSPORT_MODE_TO_MAPBOX = {
'car': 'driving',

41
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -224,6 +224,7 @@ class IsolinesRoutingConfig(ServiceConfig):
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
MAPBOX_ISO_PROVIDER = 'mapbox'
TOMTOM_PROVIDER = 'tomtom'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
@ -258,6 +259,9 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER:
self._mapbox_iso_isolines_api_keys = self._db_config.mapbox_iso_isolines_api_keys
self._mapbox_iso_isolines_service_params = db_config.mapbox_iso_isolines_service_params
elif self._isolines_provider == self.TOMTOM_PROVIDER:
self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys
self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params
@ -270,6 +274,8 @@ class IsolinesRoutingConfig(ServiceConfig):
return 'mapzen_isolines'
elif self._isolines_provider == self.MAPBOX_PROVIDER:
return 'mapbox_isolines'
elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER:
return 'mapbox_iso_isolines'
elif self._isolines_provider == self.TOMTOM_PROVIDER:
return 'tomtom_isolines'
@ -333,6 +339,18 @@ class IsolinesRoutingConfig(ServiceConfig):
def mapbox_provider(self):
return self._isolines_provider == self.MAPBOX_PROVIDER
@property
def mapbox_iso_isolines_api_keys(self):
return self._mapbox_iso_isolines_api_keys
@property
def mapbox_iso_isolines_service_params(self):
return self._mapbox_iso_isolines_service_params
@property
def mapbox_iso_provider(self):
return self._isolines_provider == self.MAPBOX_ISO_PROVIDER
@property
def tomtom_isolines_api_keys(self):
return self._tomtom_isolines_api_keys
@ -599,6 +617,7 @@ class ServicesDBConfig:
self._get_here_config()
self._get_mapzen_config()
self._get_mapbox_config()
self._get_mapbox_iso_config()
self._get_tomtom_config()
self._get_data_observatory_config()
@ -663,6 +682,16 @@ class ServicesDBConfig:
self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
def _get_mapbox_iso_config(self):
mapbox_iso_conf_json = self._get_conf('mapbox_iso_conf')
if not mapbox_iso_conf_json:
raise ConfigException('Mapbox True Isochrones configuration missing')
mapbox_iso_conf = json.loads(mapbox_iso_conf_json)
self._mapbox_iso_isolines_api_keys = mapbox_conf['isolines']['api_keys']
self._mapbox_iso_isolines_quota = tomtom_conf['isolines']['monthly_quota']
self._mapbox_iso_isolines_service_params = tomtom_conf.get('isolines', {}).get('service', {})
def _get_tomtom_config(self):
tomtom_conf_json = self._get_conf('tomtom_conf')
if not tomtom_conf_json:
@ -812,6 +841,18 @@ class ServicesDBConfig:
def mapbox_geocoder_service_params(self):
return self._mapbox_geocoder_service_params
@property
def mapbox_iso_isolines_api_keys(self):
return self._mapbox_iso_isolines_api_keys
@property
def mapbox_iso_isolines_monthly_quota(self):
return self._mapbox_iso_isolines_quota
@property
def mapbox_iso_isolines_service_params(self):
return self._mapbox_iso_isolines_service_params
@property
def tomtom_isolines_api_keys(self):
return self._tomtom_isolines_api_keys

2
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@ -22,6 +22,7 @@ class UserMetricsService:
SERVICE_HERE_ISOLINES = 'here_isolines'
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
SERVICE_MAPBOX_ISO_ISOLINES = 'mapbox_iso_isolines'
SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines'
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
@ -39,6 +40,7 @@ class UserMetricsService:
if service_type in [self.SERVICE_HERE_ISOLINES,
self.SERVICE_MAPZEN_ISOLINES,
self.SERVICE_MAPBOX_ISOLINES,
self.SERVICE_MAPBOX_ISO_ISOLINES,
self.SERVICE_TOMTOM_ISOLINES]:
return self.__used_isolines_quota(service_type, date)
elif service_type in [self.SERVICE_MAPZEN_ROUTING,

123
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py Normal file
View File

@ -0,0 +1,123 @@
from dateutil.parser import parse as date_parse
from cartodb_services.refactor.service.utils import round_robin
from cartodb_services.mapbox.types import MAPBOX_ISOLINES_APIKEY_ROUNDROBIN
class MapboxTrueIsolinesConfig(object):
"""
Configuration needed to operate the Mapbox directions service.
"""
def __init__(self,
isolines_quota,
soft_isolines_limit,
period_end_date,
cost_per_hit,
log_path,
mapbox_api_keys,
username,
organization,
service_params,
GD):
self._isolines_quota = isolines_quota
self._soft_isolines_limit = soft_isolines_limit
self._period_end_date = period_end_date
self._cost_per_hit = cost_per_hit
self._log_path = log_path
self._mapbox_api_keys = mapbox_api_keys
self._username = username
self._organization = organization
self._service_params = service_params
self._GD = GD
@property
def service_type(self):
return 'mapbox_isolines'
@property
def provider(self):
return 'mapbox'
@property
def is_high_resolution(self):
return True
@property
def isolines_quota(self):
return self._isolines_quota
@property
def soft_isolines_limit(self):
return self._soft_isolines_limit
@property
def period_end_date(self):
return self._period_end_date
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def log_path(self):
return self._log_path
@property
def mapbox_api_key(self):
return round_robin(self._mapbox_api_keys, self._GD,
MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN)
@property
def username(self):
return self._username
@property
def organization(self):
return self._organization
@property
def service_params(self):
return self._service_params
class MapboxTrueIsolinesConfigBuilder(object):
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
self._server_conf = server_conf
self._user_conf = user_conf
self._org_conf = org_conf
self._username = username
self._orgname = orgname
self._GD = GD
def get(self):
mapbox_server_conf = self._server_conf.get('mapbox_iso_conf')
mapbox_api_keys = mapbox_server_conf['isolines']['api_keys']
mapbox_service_params = mapbox_server_conf['isolines'].get('service', {})
isolines_quota = self._get_quota()
soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true'
cost_per_hit = 0
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
period_end_date = date_parse(period_end_date_str)
logger_conf = self._server_conf.get('logger_conf')
log_path = logger_conf.get('isolines_log_path', None)
return MapboxTrueIsolinesConfig(isolines_quota,
soft_isolines_limit,
period_end_date,
cost_per_hit,
log_path,
mapbox_api_keys,
self._username,
self._orgname,
mapbox_service_params,
self._GD)
def _get_quota(self):
isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota')
if isolines_quota is '':
return 0
return int(isolines_quota)

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.21.4',
version='0.23.0',
description='CartoDB Services API Python Library',

4
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -174,7 +174,7 @@ class TestGeocoderOrgConfig(TestCase):
class TestIsolinesUserConfig(TestCase):
# Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom', 'mapbox_iso']
def setUp(self):
self.redis_conn = MockRedis()
@ -190,6 +190,8 @@ class TestIsolinesUserConfig(TestCase):
assert isolines_config.service_type is 'mapzen_isolines'
elif isolines_provider is 'mapbox':
assert isolines_config.service_type is 'mapbox_isolines'
elif isolines_provider is 'mapbox_iso':
assert isolines_config.service_type is 'mapbox_iso_isolines'
elif isolines_provider is 'tomtom':
assert isolines_config.service_type is 'tomtom_isolines'
else:

1
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -77,6 +77,7 @@ def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])

33
server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py Normal file
View File

@ -0,0 +1,33 @@
import unittest
from mock import Mock
from cartodb_services.mapbox.true_isolines import MapboxTrueIsolines, DEFAULT_PROFILE
from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key
VALID_ORIGIN = Coordinate(-73.989, 40.733)
class MapboxTrueIsolinesTestCase(unittest.TestCase):
def setUp(self):
self.mapbox_isolines = MapboxTrueIsolines(apikey=mapbox_api_key(),
logger=Mock())
def test_calculate_isochrone(self):
time_ranges = [300, 900]
solution = self.mapbox_isolines.calculate_isochrone(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
time_ranges=time_ranges)
assert solution
def test_calculate_isodistance(self):
distance_range = 10000
solution = self.mapbox_isolines.calculate_isodistance(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
distance_range=distance_range)
assert solution

17
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -178,6 +178,23 @@ class TestQuotaService(TestCase):
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_mapbox_iso_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user',
provider='mapbox_iso')
qs.increment_isolines_service_use()
assert qs.check_user_quota() is True
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_org_mapbox_iso_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user',
provider='mapbox_iso',
orgname='testorg')
qs.increment_isolines_service_use()
assert qs.check_user_quota() is True
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
# Quick workaround so we don't take into account numer of credits
# spent for users that have defined the quota.
# See https://github.com/CartoDB/bigmetadata/issues/215