cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #572 from CartoDB/Add_Mapbox_true_isochrones

Browse Source 
[leapfrog] First implementation of the true Mapbox isochrones
This commit is contained in:
Antonio Carlón 2020-02-12 11:26:15 +01:00 committed by GitHub
commit 426800df63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 4275 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
server/extension/cdb_dataservices_server--0.36.0--0.37.0.sql Normal file
View File

@ -0,0 +1,126 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
if resp:
result = []
for isochrone in resp:
result_polygon = coordinates_to_polygon(isochrone.coordinates)
if result_polygon:
result.append([source, isochrone.duration, result_polygon])
else:
result.append([source, isochrone.duration, None])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(result))
return result
else:
service_manager.quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isochrones')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

129
server/extension/cdb_dataservices_server--0.37.0--0.36.0.sql Normal file
View File

@ -0,0 +1,129 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
if resp:
result = []
for isochrone in resp:
result_polygon = coordinates_to_polygon(isochrone.coordinates)
if result_polygon:
result.append([source, isochrone.duration, result_polygon])
else:
result.append([source, isochrone.duration, None])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(result))
return result
else:
service_manager.quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isochrones')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

3886
server/extension/cdb_dataservices_server--0.37.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

2
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.36.0'
default_version = '0.37.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.35.1--0.36.0.sql → server/extension/old_versions/cdb_dataservices_server--0.35.1--0.36.0.sql
View File

0
server/extension/cdb_dataservices_server--0.36.0--0.35.1.sql → server/extension/old_versions/cdb_dataservices_server--0.36.0--0.35.1.sql
View File

0
server/extension/cdb_dataservices_server--0.36.0.sql → server/extension/old_versions/cdb_dataservices_server--0.36.0.sql
View File

12
server/extension/sql/80_isolines_helper.sql
View File

@ -132,7 +132,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox import MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
@ -144,8 +144,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@ -169,7 +168,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
@ -322,7 +321,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox import MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
@ -335,8 +334,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']

2
server/extension/test/expected/00_install_test.out
View File

@ -39,7 +39,7 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi
(1 row)
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
cdb_conf_setconf
------------------

2
server/extension/test/sql/00_install_test.sql
View File

@ -16,7 +16,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

1
server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py
View File

@ -2,4 +2,3 @@ from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder
from bulk_geocoder import MapboxBulkGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse
from matrix_client import MapboxMatrixClient

241
server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py
View File

@ -1,171 +1,142 @@
'''
Python implementation for Mapbox services based isolines.
Uses the Mapbox Time Matrix service.
'''
import json
import requests
from uritemplate import URITemplate
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate
from cartodb_services.tools.spherical import (get_angles,
calculate_dest_location)
from cartodb_services.mapbox.matrix_client import (validate_profile,
DEFAULT_PROFILE,
PROFILE_WALKING,
PROFILE_DRIVING,
PROFILE_CYCLING,
ENTRY_DURATIONS,
ENTRY_DESTINATIONS,
ENTRY_LOCATION)
BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
PROFILE_DRIVING = 'driving'
PROFILE_CYCLING = 'cycling'
PROFILE_WALKING = 'walking'
DEFAULT_PROFILE = PROFILE_DRIVING
MAX_TIME_RANGE = 60 * 60 # The maximum time that can be specified is 60 minutes.
# https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location
MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed
PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed
PROFILE_DRIVING: 38.89 # In m/s, assuming 140km/h max speed
}
DEFAULT_NUM_ANGLES = 24
DEFAULT_MAX_ITERS = 5
DEFAULT_TOLERANCE = 0.1
VALID_PROFILES = (PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING)
MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES
MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS
MATRIX_TOLERANCE = DEFAULT_TOLERANCE
UNIT_FACTOR_ISOCHRONE = 1.0
UNIT_FACTOR_ISODISTANCE = 1000.0
DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE
ENTRY_FEATURES = 'features'
ENTRY_GEOMETRY = 'geometry'
ENTRY_COORDINATES = 'coordinates'
class MapboxIsolines():
'''
Python wrapper for Mapbox services based isolines.
Python wrapper for Mapbox based isolines.
'''
def __init__(self, matrix_client, logger, service_params=None):
def __init__(self, apikey, logger, service_params=None):
service_params = service_params or {}
self._matrix_client = matrix_client
self._apikey = apikey
self._logger = logger
def _calculate_matrix_cost(self, origin, targets, isorange,
profile=DEFAULT_PROFILE,
unit_factor=UNIT_FACTOR_ISOCHRONE,
number_of_angles=MATRIX_NUM_ANGLES):
response = self._matrix_client.matrix([origin] + targets,
profile)
def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
uri = URITemplate(BASEURI).expand(apikey=self._apikey,
coordinates=origin,
contours_minutes=time_range,
profile=profile)
return uri
def _validate_profile(self, profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
def _validate_time_ranges(self, time_ranges):
for time_range in time_ranges:
if time_range > MAX_TIME_RANGE:
raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format(
max_time_range=MAX_TIME_RANGE))
def _parse_coordinates(self, boundary):
coordinates = boundary.get(ENTRY_COORDINATES, [])
return [Coordinate(c[0], c[1]) for c in coordinates]
def _parse_isochrone_service(self, response):
json_response = json.loads(response)
if not json_response:
return []
costs = [None] * number_of_angles
destinations = [None] * number_of_angles
coordinates = []
if json_response:
for feature in json_response[ENTRY_FEATURES]:
geometry = feature[ENTRY_GEOMETRY]
coordinates.append(self._parse_coordinates(geometry))
for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
if cost:
costs[idx] = cost * unit_factor
return coordinates
@qps_retry(qps=5, provider='mapbox')
def _calculate_isoline(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
self._validate_time_ranges(time_ranges)
origin = '{lon},{lat}'.format(lat=origin.latitude,
lon=origin.longitude)
time_ranges.sort()
time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
uri = self._uri(origin, time_ranges_seconds, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
isolines = []
coordinates = self._parse_isochrone_service(response.text)
for t, c in zip(time_ranges, coordinates):
isolines.append(MapboxIsochronesResponse(c, t))
return isolines
elif response.status_code == requests.codes.bad_request:
return []
elif response.status_code == requests.codes.unprocessable_entity:
return []
else:
costs[idx] = isorange
for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
destination[ENTRY_LOCATION][1])
return costs, destinations
raise ServiceException(response.status_code, response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapbox isochrone service',
te)
raise ServiceException('Error getting isochrone data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox isochrone service',
exception=ce)
return []
def calculate_isochrone(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
validate_profile(profile)
self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile]
isochrones = []
for time_range in time_ranges:
upper_rmax = max_speed * time_range # an upper bound for the radius
coordinates = self.calculate_isoline(origin=origin,
isorange=time_range,
upper_rmax=upper_rmax,
cost_method=self._calculate_matrix_cost,
profile=profile,
unit_factor=UNIT_FACTOR_ISOCHRONE,
number_of_angles=MATRIX_NUM_ANGLES,
max_iterations=MATRIX_MAX_ITERS,
tolerance=MATRIX_TOLERANCE)
isochrones.append(MapboxIsochronesResponse(coordinates,
time_range))
return isochrones
return self._calculate_isoline(origin=origin,
time_ranges=time_ranges,
profile=profile)
def calculate_isodistance(self, origin, distance_range,
profile=DEFAULT_PROFILE):
validate_profile(profile)
self._validate_profile(profile)
max_speed = MAX_SPEEDS[profile]
time_range = distance_range / max_speed
return self.calculate_isochrone(origin=origin,
time_ranges=[time_range],
profile=profile)[0].coordinates
def calculate_isoline(self, origin, isorange, upper_rmax,
cost_method=_calculate_matrix_cost,
profile=DEFAULT_PROFILE,
unit_factor=DEFAULT_UNIT_FACTOR,
number_of_angles=DEFAULT_NUM_ANGLES,
max_iterations=DEFAULT_MAX_ITERS,
tolerance=DEFAULT_TOLERANCE):
# Formally, a solution is an array of {angle, radius, lat, lon, cost}
# with cardinality number_of_angles
# we're looking for a solution in which
# abs(cost - isorange) / isorange <= TOLERANCE
# Initial setup
angles = get_angles(number_of_angles)
rmax = [upper_rmax] * number_of_angles
rmin = [0.0] * number_of_angles
location_estimates = [calculate_dest_location(origin, a,
upper_rmax / 2.0)
for a in angles]
# Iterate to refine the first solution
for i in xrange(0, max_iterations):
# Calculate the "actual" cost for each location estimate.
# NOTE: sometimes it cannot calculate the cost and returns None.
# Just assume isorange and stop the calculations there
costs, destinations = cost_method(origin=origin,
targets=location_estimates,
isorange=isorange,
profile=profile,
unit_factor=unit_factor,
number_of_angles=number_of_angles)
if not costs:
continue
errors = [(cost - isorange) / float(isorange) for cost in costs]
max_abs_error = max([abs(e) for e in errors])
if max_abs_error <= tolerance:
# good enough, stop there
break
# let's refine the solution, binary search
for j in xrange(0, number_of_angles):
if abs(errors[j]) > tolerance:
if errors[j] > 0:
rmax[j] = (rmax[j] + rmin[j]) / 2.0
else:
rmin[j] = (rmax[j] + rmin[j]) / 2.0
location_estimates[j] = calculate_dest_location(origin,
angles[j],
(rmax[j] + rmin[j]) / 2.0)
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c != isorange and c < isorange * (1 + tolerance):
location_estimates_filtered.append(destinations[i])
return location_estimates_filtered
return self._calculate_isoline(origin=origin,
time_ranges=[time_range],
profile=profile)[0].coordinates
class MapboxIsochronesResponse:

92
server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py
View File

@ -1,92 +0,0 @@
'''
Python client for the Mapbox Time Matrix service.
'''
import requests
from cartodb_services.metrics import Traceable
from cartodb_services.tools.coordinates import (validate_coordinates,
marshall_coordinates)
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
'{coordinates}'
'?access_token={token}'
'&sources=0' # Set the first coordinate as source...
'&destinations=all') # ...and the rest as destinations
NUM_COORDINATES_MIN = 2 # https://www.mapbox.com/api-documentation/#matrix
NUM_COORDINATES_MAX = 25 # https://www.mapbox.com/api-documentation/#matrix
PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
PROFILE_DRIVING = 'driving'
PROFILE_CYCLING = 'cycling'
PROFILE_WALKING = 'walking'
DEFAULT_PROFILE = PROFILE_DRIVING
VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING]
ENTRY_DURATIONS = 'durations'
ENTRY_DESTINATIONS = 'destinations'
ENTRY_LOCATION = 'location'
def validate_profile(profile):
if profile not in VALID_PROFILES:
raise ValueError('{profile} is not a valid profile. '
'Valid profiles are: {valid_profiles}'.format(
profile=profile,
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
class MapboxMatrixClient(Traceable):
'''
Python wrapper for the Mapbox Time Matrix service.
'''
def __init__(self, token, logger, service_params=None):
service_params = service_params or {}
self._token = token
self._logger = logger
def _uri(self, coordinates, profile=DEFAULT_PROFILE):
return BASEURI.format(profile=profile, coordinates=coordinates,
token=self._token)
@qps_retry(qps=1)
def matrix(self, coordinates, profile=DEFAULT_PROFILE):
validate_profile(profile)
validate_coordinates(coordinates,
NUM_COORDINATES_MIN, NUM_COORDINATES_MAX)
coords = marshall_coordinates(coordinates)
uri = self._uri(coords, profile)
try:
response = requests.get(uri)
if response.status_code == requests.codes.ok:
return response.text
elif response.status_code == requests.codes.bad_request:
return '{}'
elif response.status_code == requests.codes.unprocessable_entity:
return '{}'
else:
raise ServiceException(response.status_code, response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapbox matrix service',
te)
raise ServiceException('Error getting matrix data from Mapbox',
None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox matrix service',
exception=ce)
return '{}'

3
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -676,6 +676,9 @@ class ServicesDBConfig:
raise ConfigException('Mapbox configuration missing')
mapbox_conf = json.loads(mapbox_conf_json)
# Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
# to `isolines` to ensure retrocompatibility
self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})

3
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py
View File

@ -92,6 +92,9 @@ class MapboxIsolinesConfigBuilder(object):
def get(self):
mapbox_server_conf = self._server_conf.get('mapbox_conf')
# Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
# to `isolines` to ensure retrocompatibility
mapbox_api_keys = mapbox_server_conf['matrix']['api_keys']
mapbox_service_params = mapbox_server_conf['matrix'].get('service', {})

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.22.0',
version='0.23.0',
description='CartoDB Services API Python Library',

19
server/lib/python/cartodb_services/test/test_mapboxisoline.py
View File

@ -1,20 +1,27 @@
import unittest
from mock import Mock
from cartodb_services.mapbox.isolines import MapboxIsolines
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
from cartodb_services.mapbox.isolines import MapboxIsolines, DEFAULT_PROFILE
from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key
VALID_ORIGIN = Coordinate(-73.989, 40.733)
@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
class MapboxIsolinesTestCase(unittest.TestCase):
def setUp(self):
matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock())
self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock())
self.mapbox_isolines = MapboxIsolines(apikey=mapbox_api_key(),
logger=Mock())
def test_invalid_time_range(self):
time_ranges = [4000]
with self.assertRaises(ValueError):
solution = self.mapbox_isolines.calculate_isochrone(
origin=VALID_ORIGIN,
profile=DEFAULT_PROFILE,
time_ranges=time_ranges)
def test_calculate_isochrone(self):
time_ranges = [300, 900]

58
server/lib/python/cartodb_services/test/test_mapboxmatrix.py
View File

@ -1,58 +0,0 @@
import unittest
from mock import Mock
from cartodb_services.mapbox import MapboxMatrixClient
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools import Coordinate
from credentials import mapbox_api_key
INVALID_TOKEN = 'invalid_token'
VALID_ORIGIN = Coordinate(-73.989, 40.733)
VALID_TARGET = Coordinate(-74, 40.733)
VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET]
NUM_COORDINATES_MAX = 25
INVALID_COORDINATES_EMPTY = []
INVALID_COORDINATES_MIN = [VALID_ORIGIN]
INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \
[VALID_TARGET
for x in range(0, NUM_COORDINATES_MAX + 1)]
VALID_PROFILE = DEFAULT_PROFILE
INVALID_PROFILE = 'invalid_profile'
@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
class MapboxMatrixTestCase(unittest.TestCase):
def setUp(self):
self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),
logger=Mock())
def test_invalid_profile(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(VALID_COORDINATES,
INVALID_PROFILE)
def test_invalid_coordinates_empty(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_EMPTY,
VALID_PROFILE)
def test_invalid_coordinates_max(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_MAX,
VALID_PROFILE)
def test_invalid_coordinates_min(self):
with self.assertRaises(ValueError):
self.matrix_client.matrix(INVALID_COORDINATES_MIN,
VALID_PROFILE)
def test_invalid_token(self):
invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock())
with self.assertRaises(ServiceException):
invalid_matrix.matrix(VALID_COORDINATES,
VALID_PROFILE)
def test_valid_request(self):
distance_matrix = self.matrix_client.matrix(VALID_COORDINATES,
VALID_PROFILE)
assert distance_matrix