cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Deleted old mapbox isolines implementation

Browse Source
This commit is contained in:
antoniocarlon 2020-02-11 10:00:27 +01:00
parent 484a05dbd9
commit 822c574b5c
15 changed files with 138 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py
View File

@ -2,5 +2,3 @@ from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder from geocoder import MapboxGeocoder
from bulk_geocoder import MapboxBulkGeocoder from bulk_geocoder import MapboxBulkGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse from isolines import MapboxIsolines, MapboxIsochronesResponse
from true_isolines import MapboxTrueIsolines, MapboxTrueIsochronesResponse
from matrix_client import MapboxMatrixClient

241
server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py
View File

@ -1,171 +1,142 @@
'''
Python implementation for Mapbox services based isolines.
Uses the Mapbox Time Matrix service.
'''
import json import json
import requests
from uritemplate import URITemplate
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
from cartodb_services.tools.spherical import (get_angles,
calculate_dest_location) BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
from cartodb_services.mapbox.matrix_client import (validate_profile,
DEFAULT_PROFILE, PROFILE_DRIVING = 'driving'
PROFILE_WALKING, PROFILE_CYCLING = 'cycling'
PROFILE_DRIVING, PROFILE_WALKING = 'walking'
PROFILE_CYCLING, DEFAULT_PROFILE = PROFILE_DRIVING
ENTRY_DURATIONS,
ENTRY_DESTINATIONS, MAX_TIME_RANGE = 60 * 60 # The maximum time that can be specified is 60 minutes.
ENTRY_LOCATION) # https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location
MAX_SPEEDS = { MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed
PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed PROFILE_DRIVING: 38.89 # In m/s, assuming 140km/h max speed
} }
DEFAULT_NUM_ANGLES = 24 VALID_PROFILES = [PROFILE_DRIVING,
DEFAULT_MAX_ITERS = 5 PROFILE_CYCLING,
DEFAULT_TOLERANCE = 0.1 PROFILE_WALKING]
MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES ENTRY_FEATURES = 'features'
MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS ENTRY_GEOMETRY = 'geometry'
MATRIX_TOLERANCE = DEFAULT_TOLERANCE ENTRY_COORDINATES = 'coordinates'
UNIT_FACTOR_ISOCHRONE = 1.0
UNIT_FACTOR_ISODISTANCE = 1000.0
DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE
class MapboxIsolines(): class MapboxIsolines():
''' '''
Python wrapper for Mapbox services based isolines. Python wrapper for Mapbox based isolines.
''' '''
def __init__(self, matrix_client, logger, service_params=None): def __init__(self, apikey, logger, service_params=None):
service_params = service_params or {} service_params = service_params or {}
self._matrix_client = matrix_client self._apikey = apikey
self._logger = logger self._logger = logger
def _calculate_matrix_cost(self, origin, targets, isorange, def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
profile=DEFAULT_PROFILE, uri = URITemplate(BASEURI).expand(apikey=self._apikey,
unit_factor=UNIT_FACTOR_ISOCHRONE, coordinates=origin,
number_of_angles=MATRIX_NUM_ANGLES): contours_minutes=time_range,
response = self._matrix_client.matrix([origin] + targets, profile=profile)
profile) return uri
def _validate_profile(self, profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
def _validate_time_ranges(self, time_ranges):
for time_range in time_ranges:
if time_range > MAX_TIME_RANGE:
raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format(
max_time_range=MAX_TIME_RANGE))
def _parse_coordinates(self, boundary):
coordinates = boundary.get(ENTRY_COORDINATES, [])
return [Coordinate(c[0], c[1]) for c in coordinates]
def _parse_isochrone_service(self, response):
json_response = json.loads(response) json_response = json.loads(response)
if not json_response:
return []
costs = [None] * number_of_angles coordinates = []
destinations = [None] * number_of_angles if json_response:
for feature in json_response[ENTRY_FEATURES]:
geometry = feature[ENTRY_GEOMETRY]
coordinates.append(self._parse_coordinates(geometry))
for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]): return coordinates
if cost:
costs[idx] = cost * unit_factor @qps_retry(qps=5, provider='mapbox')
def _calculate_isoline(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
self._validate_time_ranges(time_ranges)
origin = '{lon},{lat}'.format(lat=origin.latitude,
lon=origin.longitude)
time_ranges.sort()
time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
uri = self._uri(origin, time_ranges_seconds, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
isolines = []
coordinates = self._parse_isochrone_service(response.text)
for t, c in zip(time_ranges, coordinates):
isolines.append(MapboxIsochronesResponse(c, t))
return isolines
elif response.status_code == requests.codes.bad_request:
return []
elif response.status_code == requests.codes.unprocessable_entity:
return []
else: else:
costs[idx] = isorange raise ServiceException(response.status_code, response)
except requests.Timeout as te:
for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]): # In case of timeout we want to stop the job because the server
destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0], # could be down
destination[ENTRY_LOCATION][1]) self._logger.error('Timeout connecting to Mapbox isochrone service',
te)
return costs, destinations raise ServiceException('Error getting isochrone data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox isochrone service',
exception=ce)
return []
def calculate_isochrone(self, origin, time_ranges, def calculate_isochrone(self, origin, time_ranges,
profile=DEFAULT_PROFILE): profile=DEFAULT_PROFILE):
validate_profile(profile) self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile] return self._calculate_isoline(origin=origin,
time_ranges=time_ranges,
isochrones = [] profile=profile)
for time_range in time_ranges:
upper_rmax = max_speed * time_range # an upper bound for the radius
coordinates = self.calculate_isoline(origin=origin,
isorange=time_range,
upper_rmax=upper_rmax,
cost_method=self._calculate_matrix_cost,
profile=profile,
unit_factor=UNIT_FACTOR_ISOCHRONE,
number_of_angles=MATRIX_NUM_ANGLES,
max_iterations=MATRIX_MAX_ITERS,
tolerance=MATRIX_TOLERANCE)
isochrones.append(MapboxIsochronesResponse(coordinates,
time_range))
return isochrones
def calculate_isodistance(self, origin, distance_range, def calculate_isodistance(self, origin, distance_range,
profile=DEFAULT_PROFILE): profile=DEFAULT_PROFILE):
validate_profile(profile) self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile] max_speed = MAX_SPEEDS[profile]
time_range = distance_range / max_speed time_range = distance_range / max_speed
return self.calculate_isochrone(origin=origin, return self._calculate_isoline(origin=origin,
time_ranges=[time_range], time_ranges=[time_range],
profile=profile)[0].coordinates profile=profile)[0].coordinates
def calculate_isoline(self, origin, isorange, upper_rmax,
cost_method=_calculate_matrix_cost,
profile=DEFAULT_PROFILE,
unit_factor=DEFAULT_UNIT_FACTOR,
number_of_angles=DEFAULT_NUM_ANGLES,
max_iterations=DEFAULT_MAX_ITERS,
tolerance=DEFAULT_TOLERANCE):
# Formally, a solution is an array of {angle, radius, lat, lon, cost}
# with cardinality number_of_angles
# we're looking for a solution in which
# abs(cost - isorange) / isorange <= TOLERANCE
# Initial setup
angles = get_angles(number_of_angles)
rmax = [upper_rmax] * number_of_angles
rmin = [0.0] * number_of_angles
location_estimates = [calculate_dest_location(origin, a,
upper_rmax / 2.0)
for a in angles]
# Iterate to refine the first solution
for i in xrange(0, max_iterations):
# Calculate the "actual" cost for each location estimate.
# NOTE: sometimes it cannot calculate the cost and returns None.
# Just assume isorange and stop the calculations there
costs, destinations = cost_method(origin=origin,
targets=location_estimates,
isorange=isorange,
profile=profile,
unit_factor=unit_factor,
number_of_angles=number_of_angles)
if not costs:
continue
errors = [(cost - isorange) / float(isorange) for cost in costs]
max_abs_error = max([abs(e) for e in errors])
if max_abs_error <= tolerance:
# good enough, stop there
break
# let's refine the solution, binary search
for j in xrange(0, number_of_angles):
if abs(errors[j]) > tolerance:
if errors[j] > 0:
rmax[j] = (rmax[j] + rmin[j]) / 2.0
else:
rmin[j] = (rmax[j] + rmin[j]) / 2.0
location_estimates[j] = calculate_dest_location(origin,
angles[j],
(rmax[j] + rmin[j]) / 2.0)
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c != isorange and c < isorange * (1 + tolerance):
location_estimates_filtered.append(destinations[i])
return location_estimates_filtered
class MapboxIsochronesResponse: class MapboxIsochronesResponse:

92
server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py
View File

@ -1,92 +0,0 @@
'''
Python client for the Mapbox Time Matrix service.
'''
import requests
from cartodb_services.metrics import Traceable
from cartodb_services.tools.coordinates import (validate_coordinates,
marshall_coordinates)
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
'{coordinates}'
'?access_token={token}'
'&sources=0' # Set the first coordinate as source...
'&destinations=all') # ...and the rest as destinations
NUM_COORDINATES_MIN = 2 # https://www.mapbox.com/api-documentation/#matrix
NUM_COORDINATES_MAX = 25 # https://www.mapbox.com/api-documentation/#matrix
PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
PROFILE_DRIVING = 'driving'
PROFILE_CYCLING = 'cycling'
PROFILE_WALKING = 'walking'
DEFAULT_PROFILE = PROFILE_DRIVING
VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING]
ENTRY_DURATIONS = 'durations'
ENTRY_DESTINATIONS = 'destinations'
ENTRY_LOCATION = 'location'
def validate_profile(profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
class MapboxMatrixClient(Traceable):
'''
Python wrapper for the Mapbox Time Matrix service.
'''
def __init__(self, token, logger, service_params=None):
service_params = service_params or {}
self._token = token
self._logger = logger
def _uri(self, coordinates, profile=DEFAULT_PROFILE):
return BASEURI.format(profile=profile, coordinates=coordinates,
token=self._token)
@qps_retry(qps=1)
def matrix(self, coordinates, profile=DEFAULT_PROFILE):
validate_profile(profile)
validate_coordinates(coordinates,
NUM_COORDINATES_MIN, NUM_COORDINATES_MAX)
coords = marshall_coordinates(coordinates)
uri = self._uri(coords, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
return response.text
elif response.status_code == requests.codes.bad_request:
return '{}'
elif response.status_code == requests.codes.unprocessable_entity:
return '{}'
else:
raise ServiceException(response.status_code, response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapbox matrix service',
te)
raise ServiceException('Error getting matrix data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox matrix service',
exception=ce)
return '{}'

154
server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py
View File

@ -1,154 +0,0 @@
import json
import requests
from uritemplate import URITemplate
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate
BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
PROFILE_DRIVING = 'driving'
PROFILE_CYCLING = 'cycling'
PROFILE_WALKING = 'walking'
DEFAULT_PROFILE = PROFILE_DRIVING
MAX_TIME_RANGE = 60 * 60 # The maximum time that can be specified is 60 minutes.
# https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location
MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed
PROFILE_DRIVING: 38.89 # In m/s, assuming 140km/h max speed
}
VALID_PROFILES = [PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING]
ENTRY_FEATURES = 'features'
ENTRY_GEOMETRY = 'geometry'
ENTRY_COORDINATES = 'coordinates'
class MapboxTrueIsolines():
'''
Python wrapper for Mapbox based isolines.
'''
def __init__(self, apikey, logger, service_params=None):
service_params = service_params or {}
self._apikey = apikey
self._logger = logger
def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
uri = URITemplate(BASEURI).expand(apikey=self._apikey,
coordinates=origin,
contours_minutes=time_range,
profile=profile)
return uri
def _validate_profile(self, profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
def _validate_time_ranges(self, time_ranges):
for time_range in time_ranges:
if time_range > MAX_TIME_RANGE:
raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format(
max_time_range=MAX_TIME_RANGE))
def _parse_coordinates(self, boundary):
coordinates = boundary.get(ENTRY_COORDINATES, [])
return [Coordinate(c[0], c[1]) for c in coordinates]
def _parse_isochrone_service(self, response):
json_response = json.loads(response)
coordinates = []
if json_response:
for feature in json_response[ENTRY_FEATURES]:
geometry = feature[ENTRY_GEOMETRY]
coordinates.append(self._parse_coordinates(geometry))
return coordinates
@qps_retry(qps=5, provider='mapbox_iso')
def _calculate_isoline(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
self._validate_time_ranges(time_ranges)
origin = '{lon},{lat}'.format(lat=origin.latitude,
lon=origin.longitude)
time_ranges.sort()
time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
uri = self._uri(origin, time_ranges_seconds, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
isolines = []
coordinates = self._parse_isochrone_service(response.text)
for t, c in zip(time_ranges, coordinates):
isolines.append(MapboxTrueIsochronesResponse(c, t))
return isolines
elif response.status_code == requests.codes.bad_request:
return []
elif response.status_code == requests.codes.unprocessable_entity:
return []
else:
raise ServiceException(response.status_code, response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapbox isochrone service',
te)
raise ServiceException('Error getting isochrone data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox isochrone service',
exception=ce)
return []
def calculate_isochrone(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
self._validate_profile(profile)
return self._calculate_isoline(origin=origin,
time_ranges=time_ranges,
profile=profile)
def calculate_isodistance(self, origin, distance_range,
profile=DEFAULT_PROFILE):
self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile]
time_range = distance_range / max_speed
return self._calculate_isoline(origin=origin,
time_ranges=[time_range],
profile=profile)[0].coordinates
class MapboxTrueIsochronesResponse:
def __init__(self, coordinates, duration):
self._coordinates = coordinates
self._duration = duration
@property
def coordinates(self):
return self._coordinates
@property
def duration(self):
return self._duration

1
server/lib/python/cartodb_services/cartodb_services/mapbox/types.py
View File

@ -1,7 +1,6 @@
MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin' MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin'
MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin' MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin'
MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin' MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin'
MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_iso_isolines_apikey_roundrobin'
TRANSPORT_MODE_TO_MAPBOX = { TRANSPORT_MODE_TO_MAPBOX = {
'car': 'driving', 'car': 'driving',

81
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -224,7 +224,6 @@ class IsolinesRoutingConfig(ServiceConfig):
GEOCODER_PROVIDER_KEY = 'geocoder_provider' GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen' MAPZEN_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox' MAPBOX_PROVIDER = 'mapbox'
MAPBOX_ISO_PROVIDER = 'mapbox_iso'
TOMTOM_PROVIDER = 'tomtom' TOMTOM_PROVIDER = 'tomtom'
HEREMAPS_PROVIDER = 'heremaps' HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = MAPBOX_PROVIDER DEFAULT_PROVIDER = MAPBOX_PROVIDER
@ -256,12 +255,8 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_PROVIDER: elif self._isolines_provider == self.MAPBOX_PROVIDER:
self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_isolinesx_api_keys = self._db_config.mapbox_isolines_api_keys
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isolines_service_params = db_config.mapbox_isolines_service_params
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER:
self._mapbox_iso_isolines_api_keys = self._db_config.mapbox_iso_isolines_api_keys
self._mapbox_iso_isolines_service_params = db_config.mapbox_iso_isolines_service_params
elif self._isolines_provider == self.TOMTOM_PROVIDER: elif self._isolines_provider == self.TOMTOM_PROVIDER:
self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys
self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params
@ -274,8 +269,6 @@ class IsolinesRoutingConfig(ServiceConfig):
return 'mapzen_isolines' return 'mapzen_isolines'
elif self._isolines_provider == self.MAPBOX_PROVIDER: elif self._isolines_provider == self.MAPBOX_PROVIDER:
return 'mapbox_isolines' return 'mapbox_isolines'
elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER:
return 'mapbox_iso_isolines'
elif self._isolines_provider == self.TOMTOM_PROVIDER: elif self._isolines_provider == self.TOMTOM_PROVIDER:
return 'tomtom_isolines' return 'tomtom_isolines'
@ -324,33 +317,17 @@ class IsolinesRoutingConfig(ServiceConfig):
return self._isolines_provider == self.MAPZEN_PROVIDER return self._isolines_provider == self.MAPZEN_PROVIDER
@property @property
def mapbox_matrix_api_keys(self): def mapbox_isolines_api_keys(self):
return self._mapbox_matrix_api_keys return self._mapbox_isolines_api_keys
@property @property
def mapbox_matrix_service_params(self): def mapbox_isolines_service_params(self):
return self._mapbox_matrix_service_params return self._mapbox_isolines_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property @property
def mapbox_provider(self): def mapbox_provider(self):
return self._isolines_provider == self.MAPBOX_PROVIDER return self._isolines_provider == self.MAPBOX_PROVIDER
@property
def mapbox_iso_isolines_api_keys(self):
return self._mapbox_iso_isolines_api_keys
@property
def mapbox_iso_isolines_service_params(self):
return self._mapbox_iso_isolines_service_params
@property
def mapbox_iso_provider(self):
return self._isolines_provider == self.MAPBOX_ISO_PROVIDER
@property @property
def tomtom_isolines_api_keys(self): def tomtom_isolines_api_keys(self):
return self._tomtom_isolines_api_keys return self._tomtom_isolines_api_keys
@ -640,7 +617,6 @@ class ServicesDBConfig:
self._get_here_config() self._get_here_config()
self._get_mapzen_config() self._get_mapzen_config()
self._get_mapbox_config() self._get_mapbox_config()
self._get_mapbox_iso_config()
self._get_tomtom_config() self._get_tomtom_config()
self._get_geocodio_config() self._get_geocodio_config()
self._get_data_observatory_config() self._get_data_observatory_config()
@ -695,10 +671,9 @@ class ServicesDBConfig:
raise ConfigException('Mapbox configuration missing') raise ConfigException('Mapbox configuration missing')
mapbox_conf = json.loads(mapbox_conf_json) mapbox_conf = json.loads(mapbox_conf_json)
self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys'] self._mapbox_isolines_api_keys = mapbox_conf['isolines']['api_keys']
self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota'] self._mapbox_isolines_quota = mapbox_conf['isolines']['monthly_quota']
self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {}) self._mapbox_isolines_service_params = mapbox_conf.get('isolines', {}).get('service', {})
self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys'] self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota'] self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {}) self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
@ -706,16 +681,6 @@ class ServicesDBConfig:
self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
def _get_mapbox_iso_config(self):
mapbox_iso_conf_json = self._get_conf('mapbox_iso_conf')
if not mapbox_iso_conf_json:
raise ConfigException('Mapbox True Isochrones configuration missing')
mapbox_iso_conf = json.loads(mapbox_iso_conf_json)
self._mapbox_iso_isolines_api_keys = mapbox_iso_conf['isolines']['api_keys']
self._mapbox_iso_isolines_quota = mapbox_iso_conf['isolines']['monthly_quota']
self._mapbox_iso_isolines_service_params = mapbox_iso_conf.get('isolines', {}).get('service', {})
def _get_tomtom_config(self): def _get_tomtom_config(self):
tomtom_conf_json = self._get_conf('tomtom_conf') tomtom_conf_json = self._get_conf('tomtom_conf')
if not tomtom_conf_json: if not tomtom_conf_json:
@ -836,20 +801,16 @@ class ServicesDBConfig:
return self._mapzen_geocoder_service_params return self._mapzen_geocoder_service_params
@property @property
def mapbox_matrix_api_keys(self): def mapbox_isolines_api_keys(self):
return self._mapbox_matrix_api_keys return self._mapbox_isolines_api_keys
@property @property
def mapbox_matrix_monthly_quota(self): def mapbox_isolines_monthly_quota(self):
return self._mapbox_matrix_quota return self._mapbox_isolines_quota
@property @property
def mapbox_matrix_service_params(self): def mapbox_isolines_service_params(self):
return self._mapbox_matrix_service_params return self._mapbox_isolines_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property @property
def mapbox_routing_api_keys(self): def mapbox_routing_api_keys(self):
@ -875,18 +836,6 @@ class ServicesDBConfig:
def mapbox_geocoder_service_params(self): def mapbox_geocoder_service_params(self):
return self._mapbox_geocoder_service_params return self._mapbox_geocoder_service_params
@property
def mapbox_iso_isolines_api_keys(self):
return self._mapbox_iso_isolines_api_keys
@property
def mapbox_iso_isolines_monthly_quota(self):
return self._mapbox_iso_isolines_quota
@property
def mapbox_iso_isolines_service_params(self):
return self._mapbox_iso_isolines_service_params
@property @property
def tomtom_isolines_api_keys(self): def tomtom_isolines_api_keys(self):
return self._tomtom_isolines_api_keys return self._tomtom_isolines_api_keys

2
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@ -22,7 +22,6 @@ class UserMetricsService:
SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_HERE_ISOLINES = 'here_isolines'
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
SERVICE_MAPBOX_ISO_ISOLINES = 'mapbox_iso_isolines'
SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines' SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines'
SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
SERVICE_MAPBOX_ROUTING = 'routing_mapbox' SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
@ -40,7 +39,6 @@ class UserMetricsService:
if service_type in [self.SERVICE_HERE_ISOLINES, if service_type in [self.SERVICE_HERE_ISOLINES,
self.SERVICE_MAPZEN_ISOLINES, self.SERVICE_MAPZEN_ISOLINES,
self.SERVICE_MAPBOX_ISOLINES, self.SERVICE_MAPBOX_ISOLINES,
self.SERVICE_MAPBOX_ISO_ISOLINES,
self.SERVICE_TOMTOM_ISOLINES]: self.SERVICE_TOMTOM_ISOLINES]:
return self.__used_isolines_quota(service_type, date) return self.__used_isolines_quota(service_type, date)
elif service_type in [self.SERVICE_MAPZEN_ROUTING, elif service_type in [self.SERVICE_MAPZEN_ROUTING,

4
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py
View File

@ -92,8 +92,8 @@ class MapboxIsolinesConfigBuilder(object):
def get(self): def get(self):
mapbox_server_conf = self._server_conf.get('mapbox_conf') mapbox_server_conf = self._server_conf.get('mapbox_conf')
mapbox_api_keys = mapbox_server_conf['matrix']['api_keys'] mapbox_api_keys = mapbox_server_conf['isolines']['api_keys']
mapbox_service_params = mapbox_server_conf['matrix'].get('service', {}) mapbox_service_params = mapbox_server_conf['isolines'].get('service', {})
isolines_quota = self._get_quota() isolines_quota = self._get_quota()
soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true'

123
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py
View File

@ -1,123 +0,0 @@
from dateutil.parser import parse as date_parse
from cartodb_services.refactor.service.utils import round_robin
from cartodb_services.mapbox.types import MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN
class MapboxTrueIsolinesConfig(object):
"""
Configuration needed to operate the Mapbox directions service.
"""
def __init__(self,
isolines_quota,
soft_isolines_limit,
period_end_date,
cost_per_hit,
log_path,
mapbox_api_keys,
username,
organization,
service_params,
GD):
self._isolines_quota = isolines_quota
self._soft_isolines_limit = soft_isolines_limit
self._period_end_date = period_end_date
self._cost_per_hit = cost_per_hit
self._log_path = log_path
self._mapbox_api_keys = mapbox_api_keys
self._username = username
self._organization = organization
self._service_params = service_params
self._GD = GD
@property
def service_type(self):
return 'mapbox_isolines'
@property
def provider(self):
return 'mapbox'
@property
def is_high_resolution(self):
return True
@property
def isolines_quota(self):
return self._isolines_quota
@property
def soft_isolines_limit(self):
return self._soft_isolines_limit
@property
def period_end_date(self):
return self._period_end_date
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def log_path(self):
return self._log_path
@property
def mapbox_api_key(self):
return round_robin(self._mapbox_api_keys, self._GD,
MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN)
@property
def username(self):
return self._username
@property
def organization(self):
return self._organization
@property
def service_params(self):
return self._service_params
class MapboxTrueIsolinesConfigBuilder(object):
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
self._server_conf = server_conf
self._user_conf = user_conf
self._org_conf = org_conf
self._username = username
self._orgname = orgname
self._GD = GD
def get(self):
mapbox_server_conf = self._server_conf.get('mapbox_iso_conf')
mapbox_api_keys = mapbox_server_conf['isolines']['api_keys']
mapbox_service_params = mapbox_server_conf['isolines'].get('service', {})
isolines_quota = self._get_quota()
soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true'
cost_per_hit = 0
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
period_end_date = date_parse(period_end_date_str)
logger_conf = self._server_conf.get('logger_conf')
log_path = logger_conf.get('isolines_log_path', None)
return MapboxTrueIsolinesConfig(isolines_quota,
soft_isolines_limit,
period_end_date,
cost_per_hit,
log_path,
mapbox_api_keys,
self._username,
self._orgname,
mapbox_service_params,
self._GD)
def _get_quota(self):
isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota')
if isolines_quota is '':
return 0
return int(isolines_quota)

4
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -180,7 +180,7 @@ class TestGeocoderOrgConfig(TestCase):
class TestIsolinesUserConfig(TestCase): class TestIsolinesUserConfig(TestCase):
# Don't test mapbox. See CartoDB/cartodb-management/issues/5199" # Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom', 'mapbox_iso'] ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
def setUp(self): def setUp(self):
self.redis_conn = MockRedis() self.redis_conn = MockRedis()
@ -196,8 +196,6 @@ class TestIsolinesUserConfig(TestCase):
assert isolines_config.service_type is 'mapzen_isolines' assert isolines_config.service_type is 'mapzen_isolines'
elif isolines_provider is 'mapbox': elif isolines_provider is 'mapbox':
assert isolines_config.service_type is 'mapbox_isolines' assert isolines_config.service_type is 'mapbox_isolines'
elif isolines_provider is 'mapbox_iso':
assert isolines_config.service_type is 'mapbox_iso_isolines'
elif isolines_provider is 'tomtom': elif isolines_provider is 'tomtom':
assert isolines_config.service_type is 'tomtom_isolines' assert isolines_config.service_type is 'tomtom_isolines'
else: else:

3
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -76,8 +76,7 @@ def increment_service_uses(redis_conn, username, orgname=None,
def plpy_mock_config(): def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])

19
server/lib/python/cartodb_services/test/test_mapboxisoline.py
View File

@ -1,20 +1,27 @@
import unittest import unittest
from mock import Mock from mock import Mock
from cartodb_services.mapbox.isolines import MapboxIsolines from cartodb_services.mapbox.isolines import MapboxIsolines, DEFAULT_PROFILE
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key from credentials import mapbox_api_key
VALID_ORIGIN = Coordinate(-73.989, 40.733) VALID_ORIGIN = Coordinate(-73.989, 40.733)
@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
class MapboxIsolinesTestCase(unittest.TestCase): class MapboxIsolinesTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock()) self.mapbox_isolines = MapboxIsolines(apikey=mapbox_api_key(),
self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock()) logger=Mock())
def test_invalid_time_range(self):
time_ranges = [4000]
with self.assertRaises(ValueError):
solution = self.mapbox_isolines.calculate_isochrone(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
time_ranges=time_ranges)
def test_calculate_isochrone(self): def test_calculate_isochrone(self):
time_ranges = [300, 900] time_ranges = [300, 900]

58
server/lib/python/cartodb_services/test/test_mapboxmatrix.py
View File

@ -1,58 +0,0 @@
import unittest
from mock import Mock
from cartodb_services.mapbox import MapboxMatrixClient
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key
INVALID_TOKEN = 'invalid_token'
VALID_ORIGIN = Coordinate(-73.989, 40.733)
VALID_TARGET = Coordinate(-74, 40.733)
VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET]
NUM_COORDINATES_MAX = 25
INVALID_COORDINATES_EMPTY = []
INVALID_COORDINATES_MIN = [VALID_ORIGIN]
INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \
[VALID_TARGET
for x in range(0, NUM_COORDINATES_MAX + 1)]
VALID_PROFILE = DEFAULT_PROFILE
INVALID_PROFILE = 'invalid_profile'
@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
class MapboxMatrixTestCase(unittest.TestCase):
def setUp(self):
self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),
logger=Mock())
def test_invalid_profile(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(VALID_COORDINATES,
INVALID_PROFILE)
def test_invalid_coordinates_empty(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_EMPTY,
VALID_PROFILE)
def test_invalid_coordinates_max(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_MAX,
VALID_PROFILE)
def test_invalid_coordinates_min(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_MIN,
VALID_PROFILE)
def test_invalid_token(self):
invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock())
with self.assertRaises(ServiceException):
invalid_matrix.matrix(VALID_COORDINATES,
VALID_PROFILE)
def test_valid_request(self):
distance_matrix = self.matrix_client.matrix(VALID_COORDINATES,
VALID_PROFILE)
assert distance_matrix

42
server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py
View File

@ -1,42 +0,0 @@
import unittest
from mock import Mock
from cartodb_services.mapbox.true_isolines import MapboxTrueIsolines, DEFAULT_PROFILE
from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key
VALID_ORIGIN = Coordinate(-73.989, 40.733)
class MapboxTrueIsolinesTestCase(unittest.TestCase):
def setUp(self):
self.mapbox_isolines = MapboxTrueIsolines(apikey=mapbox_api_key(),
logger=Mock())
def test_invalid_time_range(self):
time_ranges = [4000]
with self.assertRaises(ValueError):
solution = self.mapbox_isolines.calculate_isochrone(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
time_ranges=time_ranges)
def test_calculate_isochrone(self):
time_ranges = [300, 900]
solution = self.mapbox_isolines.calculate_isochrone(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
time_ranges=time_ranges)
assert solution
def test_calculate_isodistance(self):
distance_range = 10000
solution = self.mapbox_isolines.calculate_isodistance(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
distance_range=distance_range)
assert solution

17
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -178,23 +178,6 @@ class TestQuotaService(TestCase):
qs.increment_isolines_service_use(amount=1500000) qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False assert qs.check_user_quota() is False
def test_should_check_user_mapbox_iso_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user',
provider='mapbox_iso')
qs.increment_isolines_service_use()
assert qs.check_user_quota() is True
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_org_mapbox_iso_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user',
provider='mapbox_iso',
orgname='testorg')
qs.increment_isolines_service_use()
assert qs.check_user_quota() is True
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
# Quick workaround so we don't take into account numer of credits # Quick workaround so we don't take into account numer of credits
# spent for users that have defined the quota. # spent for users that have defined the quota.
# See https://github.com/CartoDB/bigmetadata/issues/215 # See https://github.com/CartoDB/bigmetadata/issues/215