From 893b8db374877ad29b45dfbccfa00a18ea1b8bca Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 3 Jun 2016 19:34:55 +0200 Subject: [PATCH 01/31] First stage is calculating the matrix of points --- .../cartodb_services/mapzen/__init__.py | 1 + .../cartodb_services/mapzen/isolines.py | 160 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py index ee7552c..cc651c2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -1,2 +1,3 @@ from routing import MapzenRouting, MapzenRoutingResponse +from isolines import MapzenIsolines from geocoder import MapzenGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py new file mode 100644 index 0000000..2aaa585 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -0,0 +1,160 @@ +import requests +import json +import re + +from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2 +from exceptions import WrongParams, MalformedResult +from qps import qps_retry +from cartodb_services.tools import Coordinate, PolyLine + + +class MapzenIsolines: + + 'A Mapzen Isochrones feature using the mapzen distance matrix' + + MATRIX_API_URL = 'https://matrix.mapzen.com/one_to_many' + + ACCEPTED_MODES = { + "walk": "pedestrian", + "car": "auto", + } + + ACCEPTED_TYPES = ['distance', 'time'] + + AUTO_SHORTEST = 'auto_shortest' + + OPTIONAL_PARAMS = [ + 'mode_type', + ] + + METRICS_UNITS = 'kilometers' + IMPERIAL_UNITS = 'miles' + + EARTH_RADIUS_METERS = 6371000 + EARTH_RADIUS_MILES = 3959 + + DISTANCE_MULTIPLIER = [0.8, 0.9, 1, 1.10, 1.20] # From 80% to 120% of range + METERS_PER_SECOND = { + "walk": 1.38889, #Based on 5Km/h + "car": 22.3 #Based on 80Km/h + } + UNIT_MULTIPLIER = { + "kilometers": 1, + "miles": 0.3048 + } + + def __init__(self, app_key, base_url=MATRIX_API_URL): + self._app_key = app_key + self._url = base_url + + def calculate_isochrone(self, origin, mode, mode_range=[], units=METRICS_UNITS): + return self._calculate_isolines(origin, mode, 'time', mode_range, units) + + def calculate_isodistance(self, origin, mode, mode_range=[], units=METRICS_UNITS): + return self._calculate_isolines(origin, mode, 'distance', mode_range, units) + + def _calculate_isolines(self, origin, mode, mode_type, mode_range=[], units=METRICS_UNITS): + for r in mode_range: + radius = self._calculate_radius(r, mode, mode_type, units) + destination_points = self._calculate_destination_points(origin, radius) + destination_matrix = self._calculate_destination_matrix(origin, destination_points, mode, units) + + def _calculate_radius(self, init_radius, mode, mode_type, units): + if mode_type is 'time': + radius_meters = init_radius * self.METERS_PER_SECOND[mode] * self.UNIT_MULTIPLIER[units] + else: + radius_meters = init_radius + + return [init_radius*multiplier for multiplier in self.DISTANCE_MULTIPLIER] + + def _calculate_destination_points(self, origin, radius): + destinations = [] + angles = [i*36 for i in range(10)] + for angle in angles: + d = [self._calculate_destination_point(origin, r, angle) for r in radius] + destinations.extend(d) + return destinations + + def _calculate_destination_point(self, origin, radius, angle): + bearing = radians(angle) + origin_lat_radians = radians(origin.latitude) + origin_long_radians = radians(origin.longitude) + dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(bearing)) + dest_lng_radians = origin_long_radians + atan2(sin(bearing) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians)) + + return Coordinate(degrees(dest_lng_radians), degrees(dest_lat_radians)) + + def _calculate_destination_matrix(self, origin, destination_points, mode, units): + json_request_params = self.__parse_json_parameters(destination_points, mode, units) + request_params = self.__parse_request_parameters(json_request_params) + response = requests.get(self._url, params=request_params) + import ipdb; ipdb.set_trace() # breakpoint 2b65ce71 // + if response.status_code == requests.codes.ok: + return self.__parse_routing_response(response.text) + elif response.status_code == requests.codes.bad_request: + return MapzenIsochronesResponse(None, None, None) + else: + response.raise_for_status() + + def __parse_request_parameters(self, json_request): + request_options = {"json": json_request} + request_options.update({'api_key': self._app_key}) + + return request_options + + def __parse_json_parameters(self, destination_points, mode, units): + import ipdb; ipdb.set_trace() # breakpoint 2b65ce71 // + json_options = {"locations": self._parse_destination_points(destination_points)} + json_options.update({'costing': self.ACCEPTED_MODES[mode]}) + #json_options.update({"directions_options": {'units': units, + # 'narrative': False}}) + + return json.dumps(json_options) + + def _parse_destination_points(self, destination_points): + destinations = [] + for dest in destination_points: + destinations.append({"lat": dest.latitude, "lon": dest.longitude}) + + return destinations + + + def __parse_matrix_response(self, response): + try: + parsed_json_response = json.loads(response) + except IndexError: + return [] + except KeyError: + raise MalformedResult() + + def __parse_mode_param(self, mode, options): + if mode in self.ACCEPTED_MODES: + mode_source = self.ACCEPTED_MODES[mode] + else: + raise WrongParams("{0} is not an accepted mode type".format(mode)) + + if mode == self.ACCEPTED_MODES['car'] and 'mode_type' in options and \ + options['mode_type'] == 'shortest': + mode = self.AUTO_SHORTEST + + return mode + + +class MapzenIsochronesResponse: + + def __init__(self, shape, length, duration): + self._shape = shape + self._length = length + self._duration = duration + + @property + def shape(self): + return self._shape + + @property + def length(self): + return self._length + + @property + def duration(self): + return self._duration From 40cacd99dc2fec21673009f05ef1a91a5b250c1c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 4 Jul 2016 20:07:58 +0200 Subject: [PATCH 02/31] Some code trying to pull everything together (WIP) --- server/extension/sql/80_isolines_helper.sql | 68 +++++++++++++++++++++ server/extension/sql/90_isochrone.sql | 25 ++++++++ 2 files changed, 93 insertions(+) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index b598cf6..f0e4e57 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -53,3 +53,71 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_routing_isolines( + username TEXT, + orgname TEXT, + isotype TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon # TODO do we use the same types? + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + #quota_service = QuotaService(user_isolines_routing_config, redis_conn) + #if not quota_service.check_user_quota(): + # plpy.error('You have reached the limit of your quota') + + try: + # TODO: encapsulate or refactor this ugly code + mapzen_conf_str = plpy.execute("SELECT * FROM CDB_Conf_Getconf('mapzen_conf') AS mapzen_conf")[0]['mapzen_conf'] + mapzen_conf = json.loads(mapzen_conf_str) + + client = MapzenIsolines(mapzen_conf['routing']['api_key']) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if isotype == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif isotype == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + #quota_service.increment_success_service_use() + #quota_service.increment_isolines_service_use(len(resp)) + return result + else: + #quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + #quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) + #plpy.notice(traceback.format_tb(traceback_)) + raise e + #plpy.error(error_msg) + finally: + pass + #quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 6237bd7..a75ec67 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -19,3 +19,28 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return isolines $$ LANGUAGE plpythonu; + + +-- mapzen isochrones +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + # if we were to add a config check, it'll go here + #if user_isolines_config.google_services_user: + # plpy.error('This service is not available for google service users.') + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; From 53fe4ce21d5f55d4d2ce4a68a708b798b0dfd925 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 10:32:38 +0200 Subject: [PATCH 03/31] An attempt to adapt paremetrs (WIP) --- server/extension/sql/80_isolines_helper.sql | 7 ++++--- .../cartodb_services/cartodb_services/mapzen/isolines.py | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index f0e4e57..43ff444 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -91,10 +91,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ else: source_str = None + # -- TODO Support options properly if isotype == 'isodistance': - resp = client.calculate_isodistance(source_str, mode, data_range, options) + resp = client.calculate_isodistance(source_str, mode, data_range) elif isotype == 'isochrone': - resp = client.calculate_isochrone(source_str, mode, data_range, options) + resp = client.calculate_isochrone(source_str, mode, data_range) if resp: result = [] @@ -114,7 +115,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ type_, value_, traceback_ = sys.exc_info() #quota_service.increment_failed_service_use() error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) - #plpy.notice(traceback.format_tb(traceback_)) + plpy.debug(traceback.format_tb(traceback_)) raise e #plpy.error(error_msg) finally: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 2aaa585..1d2a814 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -60,7 +60,14 @@ class MapzenIsolines: destination_matrix = self._calculate_destination_matrix(origin, destination_points, mode, units) def _calculate_radius(self, init_radius, mode, mode_type, units): + import logging + #logging.basicConfig(filename='/tmp/isolines.log',level=logging.DEBUG) + logging.debug(mode_type) if mode_type is 'time': + logging.debug(init_radius) + logging.debug(self.METERS_PER_SECOND[mode]) + logging.debug("units = %s", units) + logging.debug(self.UNIT_MULTIPLIER[units]) radius_meters = init_radius * self.METERS_PER_SECOND[mode] * self.UNIT_MULTIPLIER[units] else: radius_meters = init_radius From a70560e566941445cce469b1821314b33fc8fa8f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 11:12:15 +0200 Subject: [PATCH 04/31] Minimal Mapzen Time-Distance Matrix client --- .../cartodb_services/mapzen/__init__.py | 1 + .../cartodb_services/mapzen/matrix_client.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py index cc651c2..e3dca35 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -1,3 +1,4 @@ from routing import MapzenRouting, MapzenRoutingResponse from isolines import MapzenIsolines from geocoder import MapzenGeocoder +from matrix_client import MatrixClient diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py new file mode 100644 index 0000000..0c2d1a8 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -0,0 +1,29 @@ +import requests +import json + +class MatrixClient: + + ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many' + + def __init__(self, matrix_key): + self._matrix_key = matrix_key + + """Get distances and times to a set of locations. + See https://mapzen.com/documentation/matrix/api-reference/ + + Args: + locations Array of {lat: y, lon: x} + costing Costing model to use + + Returns: + A dict with one_to_many, units and locations + """ + def one_to_many(self, locations, costing): + request_params = { + 'json': json.dumps({'locations': locations}), + 'costing': costing, + 'api_key': self._matrix_key + } + response = requests.get(self.ONE_TO_MANY_URL, params=request_params) + + return response.json() From 96199b0d6dfea835d6bb23bc87060e5732ef4094 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 16:18:59 +0200 Subject: [PATCH 05/31] Add example to code doc --- .../cartodb_services/mapzen/matrix_client.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 0c2d1a8..de9b5cc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -3,6 +3,18 @@ import json class MatrixClient: + """ + A minimal client for Mapzen Time-Distance Matrix Service + + Example: + + client = MatrixClient('your_api_key') + locations = [{"lat":40.744014,"lon":-73.990508},{"lat":40.739735,"lon":-73.979713},{"lat":40.752522,"lon":-73.985015},{"lat":40.750117,"lon":-73.983704},{"lat":40.750552,"lon":-73.993519}] + costing = 'pedestrian' + + print client.one_to_many(locations, costing) + """ + ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many' def __init__(self, matrix_key): @@ -17,7 +29,7 @@ class MatrixClient: Returns: A dict with one_to_many, units and locations - """ + """ def one_to_many(self, locations, costing): request_params = { 'json': json.dumps({'locations': locations}), From 46971fe96f45db7aa5da7a0e8e0ac63f74d375f4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 16:19:19 +0200 Subject: [PATCH 06/31] Raise error when response not OK --- .../cartodb_services/cartodb_services/mapzen/matrix_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index de9b5cc..9b4df53 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -38,4 +38,6 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params) + response.raise_for_status() # raise exception if not 200 OK + return response.json() From 87413255af97f71fb735c68387dcfbeb6da03cb5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 16:19:54 +0200 Subject: [PATCH 07/31] Major rewrite of MapzenIsolines (WIP) --- .../cartodb_services/mapzen/isolines.py | 194 +++++------------- 1 file changed, 47 insertions(+), 147 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 1d2a814..76e5b96 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -1,167 +1,67 @@ -import requests -import json -import re - from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2 -from exceptions import WrongParams, MalformedResult -from qps import qps_retry -from cartodb_services.tools import Coordinate, PolyLine - class MapzenIsolines: - 'A Mapzen Isochrones feature using the mapzen distance matrix' - - MATRIX_API_URL = 'https://matrix.mapzen.com/one_to_many' - - ACCEPTED_MODES = { - "walk": "pedestrian", - "car": "auto", - } - - ACCEPTED_TYPES = ['distance', 'time'] - - AUTO_SHORTEST = 'auto_shortest' - - OPTIONAL_PARAMS = [ - 'mode_type', - ] - - METRICS_UNITS = 'kilometers' - IMPERIAL_UNITS = 'miles' + NUMBER_OF_ANGLES = 12 + MAX_ITERS = 5 + TOLERANCE = 0.1 EARTH_RADIUS_METERS = 6371000 - EARTH_RADIUS_MILES = 3959 - DISTANCE_MULTIPLIER = [0.8, 0.9, 1, 1.10, 1.20] # From 80% to 120% of range - METERS_PER_SECOND = { - "walk": 1.38889, #Based on 5Km/h - "car": 22.3 #Based on 80Km/h - } - UNIT_MULTIPLIER = { - "kilometers": 1, - "miles": 0.3048 - } + def __init__(self, matrix_client): + self._matrix_client = matrix_client - def __init__(self, app_key, base_url=MATRIX_API_URL): - self._app_key = app_key - self._url = base_url + """Get an isochrone using mapzen API. - def calculate_isochrone(self, origin, mode, mode_range=[], units=METRICS_UNITS): - return self._calculate_isolines(origin, mode, 'time', mode_range, units) + The implementation tries to sick close to the SQL API: + cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline - def calculate_isodistance(self, origin, mode, mode_range=[], units=METRICS_UNITS): - return self._calculate_isolines(origin, mode, 'distance', mode_range, units) + But this calculates just one isoline. - def _calculate_isolines(self, origin, mode, mode_type, mode_range=[], units=METRICS_UNITS): - for r in mode_range: - radius = self._calculate_radius(r, mode, mode_type, units) - destination_points = self._calculate_destination_points(origin, radius) - destination_matrix = self._calculate_destination_matrix(origin, destination_points, mode, units) + Args: + origin dict containing {lat: y, lon: x} + transport_mode string, for the moment just "car" or "walk" + isorange int range of the isoline in seconds - def _calculate_radius(self, init_radius, mode, mode_type, units): - import logging - #logging.basicConfig(filename='/tmp/isolines.log',level=logging.DEBUG) - logging.debug(mode_type) - if mode_type is 'time': - logging.debug(init_radius) - logging.debug(self.METERS_PER_SECOND[mode]) - logging.debug("units = %s", units) - logging.debug(self.UNIT_MULTIPLIER[units]) - radius_meters = init_radius * self.METERS_PER_SECOND[mode] * self.UNIT_MULTIPLIER[units] - else: - radius_meters = init_radius + Returns: + Array of {lon: x, lat: y} as a representation of the isoline + """ + def calculate_isochrone(self, origin, transport_mode, isorange): + if transport_mode != 'walk': + # TODO move this restriction to the appropriate place + raise NotImplementedError('walk is the only supported mode for the moment') - return [init_radius*multiplier for multiplier in self.DISTANCE_MULTIPLIER] + bearings = self._get_bearings(self.NUMBER_OF_ANGLES) + location_estimates = [self._get_dest_location_estimate(origin, b, isorange) for b in bearings] - def _calculate_destination_points(self, origin, radius): - destinations = [] - angles = [i*36 for i in range(10)] - for angle in angles: - d = [self._calculate_destination_point(origin, r, angle) for r in radius] - destinations.extend(d) - return destinations - - def _calculate_destination_point(self, origin, radius, angle): - bearing = radians(angle) - origin_lat_radians = radians(origin.latitude) - origin_long_radians = radians(origin.longitude) - dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(bearing)) - dest_lng_radians = origin_long_radians + atan2(sin(bearing) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians)) - - return Coordinate(degrees(dest_lng_radians), degrees(dest_lat_radians)) - - def _calculate_destination_matrix(self, origin, destination_points, mode, units): - json_request_params = self.__parse_json_parameters(destination_points, mode, units) - request_params = self.__parse_request_parameters(json_request_params) - response = requests.get(self._url, params=request_params) - import ipdb; ipdb.set_trace() # breakpoint 2b65ce71 // - if response.status_code == requests.codes.ok: - return self.__parse_routing_response(response.text) - elif response.status_code == requests.codes.bad_request: - return MapzenIsochronesResponse(None, None, None) - else: - response.raise_for_status() - - def __parse_request_parameters(self, json_request): - request_options = {"json": json_request} - request_options.update({'api_key': self._app_key}) - - return request_options - - def __parse_json_parameters(self, destination_points, mode, units): - import ipdb; ipdb.set_trace() # breakpoint 2b65ce71 // - json_options = {"locations": self._parse_destination_points(destination_points)} - json_options.update({'costing': self.ACCEPTED_MODES[mode]}) - #json_options.update({"directions_options": {'units': units, - # 'narrative': False}}) - - return json.dumps(json_options) - - def _parse_destination_points(self, destination_points): - destinations = [] - for dest in destination_points: - destinations.append({"lat": dest.latitude, "lon": dest.longitude}) - - return destinations + # calculate the "actual" cost for each location estimate as first iteration + resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') + costs = resp['one_to_many'][0][1:] + #import pdb; pdb.set_trace() - def __parse_matrix_response(self, response): - try: - parsed_json_response = json.loads(response) - except IndexError: - return [] - except KeyError: - raise MalformedResult() + # NOTE: all angles in calculations are in radians + def _get_bearings(self, number_of_angles): + step = (2.0 * pi) / number_of_angles + return [(x * step) for x in xrange(0, number_of_angles)] - def __parse_mode_param(self, mode, options): - if mode in self.ACCEPTED_MODES: - mode_source = self.ACCEPTED_MODES[mode] - else: - raise WrongParams("{0} is not an accepted mode type".format(mode)) + # TODO: this just works for walk isochrone + # TODO: split this into two + def _get_dest_location_estimate(self, origin, bearing, trange): + # my rule of thumb: normal walk speed is about 1km in 10 minutes = 6 km/h + # use 12 km/h as an upper bound + speed = 3.333333 # in m/s + distance = speed * trange - if mode == self.ACCEPTED_MODES['car'] and 'mode_type' in options and \ - options['mode_type'] == 'shortest': - mode = self.AUTO_SHORTEST + return self._calculate_dest_location(origin, bearing, distance) - return mode + def _calculate_dest_location(self, origin, angle, radius): + origin_lat_radians = radians(origin['lat']) + origin_long_radians = radians(origin['lon']) + dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(angle)) + dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians)) - -class MapzenIsochronesResponse: - - def __init__(self, shape, length, duration): - self._shape = shape - self._length = length - self._duration = duration - - @property - def shape(self): - return self._shape - - @property - def length(self): - return self._length - - @property - def duration(self): - return self._duration + return { + 'lon': degrees(dest_lng_radians), + 'lat': degrees(dest_lat_radians) + } From 9becf1adb460bf5e07f6eb168aa2ba7fa7a999b3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 17:07:43 +0200 Subject: [PATCH 08/31] Iterative part of the algorithm (WIP) --- .../cartodb_services/mapzen/isolines.py | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 76e5b96..5ecfa79 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -31,30 +31,50 @@ class MapzenIsolines: # TODO move this restriction to the appropriate place raise NotImplementedError('walk is the only supported mode for the moment') - bearings = self._get_bearings(self.NUMBER_OF_ANGLES) - location_estimates = [self._get_dest_location_estimate(origin, b, isorange) for b in bearings] + # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES + # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE + + # Initial setup + angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles + upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed + rmax = [upper_rmax] * self.NUMBER_OF_ANGLES + rmin = [0.0] * self.NUMBER_OF_ANGLES + location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles] # calculate the "actual" cost for each location estimate as first iteration resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') - costs = resp['one_to_many'][0][1:] + costs = [c['time'] for c in resp['one_to_many'][0][1:]] #import pdb; pdb.set_trace() + # iterate to refine the first solution, if needed + for i in xrange(0, self.MAX_ITERS): + errors = [(cost - isorange) / float(isorange) for cost in costs] + max_abs_error = [abs(e) for e in errors] + if max_abs_error <= self.TOLERANCE: + # good enough, stop there + break + + # let's refine the solution, binary search + for j in xrange(0, self.NUMBER_OF_ANGLES): + if errors[j] > 0: + rmax[j] = (rmax[j] - rmin[j]) / 2.0 + else: + rmin[j] = (rmax[j] - rmin[j]) / 2.0 + location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]-rmin[j])/2.0) + + # and check "actual" costs again + resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') + costs = [c['time'] for c in resp['one_to_many'][0][1:]] + + return location_estimates + + # NOTE: all angles in calculations are in radians - def _get_bearings(self, number_of_angles): + def _get_angles(self, number_of_angles): step = (2.0 * pi) / number_of_angles return [(x * step) for x in xrange(0, number_of_angles)] - # TODO: this just works for walk isochrone - # TODO: split this into two - def _get_dest_location_estimate(self, origin, bearing, trange): - # my rule of thumb: normal walk speed is about 1km in 10 minutes = 6 km/h - # use 12 km/h as an upper bound - speed = 3.333333 # in m/s - distance = speed * trange - - return self._calculate_dest_location(origin, bearing, distance) - def _calculate_dest_location(self, origin, angle, radius): origin_lat_radians = radians(origin['lat']) origin_long_radians = radians(origin['lon']) From 9a9f35d9c25e48349e97ce16773cf3a5bf358cfc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 17:41:15 +0200 Subject: [PATCH 09/31] Fix silly typos spotted by jgoizueta (WIP) --- .../cartodb_services/cartodb_services/mapzen/isolines.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 5ecfa79..0e89ef4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -57,10 +57,10 @@ class MapzenIsolines: # let's refine the solution, binary search for j in xrange(0, self.NUMBER_OF_ANGLES): if errors[j] > 0: - rmax[j] = (rmax[j] - rmin[j]) / 2.0 + rmax[j] = (rmax[j] + rmin[j]) / 2.0 else: - rmin[j] = (rmax[j] - rmin[j]) / 2.0 - location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]-rmin[j])/2.0) + rmin[j] = (rmax[j] + rmin[j]) / 2.0 + location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) # and check "actual" costs again resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') From 2d95601c5a853605c2f4d10e1c724dd6281a1f91 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 18:35:40 +0200 Subject: [PATCH 10/31] Fix: max_abs_error should be a scalar --- .../python/cartodb_services/cartodb_services/mapzen/isolines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 0e89ef4..f6f50aa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -49,7 +49,7 @@ class MapzenIsolines: # iterate to refine the first solution, if needed for i in xrange(0, self.MAX_ITERS): errors = [(cost - isorange) / float(isorange) for cost in costs] - max_abs_error = [abs(e) for e in errors] + max_abs_error = max([abs(e) for e in errors]) if max_abs_error <= self.TOLERANCE: # good enough, stop there break From 77cdc3d8ffc81470968fafeb77fa2455bbffd1f1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 18:36:28 +0200 Subject: [PATCH 11/31] Only refine individual solutions when error > TOLERANCE --- .../cartodb_services/mapzen/isolines.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index f6f50aa..2feb6ed 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -56,11 +56,13 @@ class MapzenIsolines: # let's refine the solution, binary search for j in xrange(0, self.NUMBER_OF_ANGLES): - if errors[j] > 0: - rmax[j] = (rmax[j] + rmin[j]) / 2.0 - else: - rmin[j] = (rmax[j] + rmin[j]) / 2.0 - location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) + + if abs(errors[j]) > self.TOLERANCE: + if errors[j] > 0: + rmax[j] = (rmax[j] + rmin[j]) / 2.0 + else: + rmin[j] = (rmax[j] + rmin[j]) / 2.0 + location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) # and check "actual" costs again resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') From b78bd057548219500639c4adb472547f3f198def Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 20:52:41 +0200 Subject: [PATCH 12/31] Be resilient to None cost estimation --- .../cartodb_services/mapzen/isolines.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 2feb6ed..31b7c01 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -1,4 +1,5 @@ from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2 +import logging class MapzenIsolines: @@ -27,6 +28,14 @@ class MapzenIsolines: Array of {lon: x, lat: y} as a representation of the isoline """ def calculate_isochrone(self, origin, transport_mode, isorange): + + # NOTE: not for production + #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log') + logging.debug('origin = %s' % origin) + logging.debug('transport_mode = %s' % transport_mode) + logging.debug('isorange = %d' % isorange) + + if transport_mode != 'walk': # TODO move this restriction to the appropriate place raise NotImplementedError('walk is the only supported mode for the moment') @@ -42,12 +51,14 @@ class MapzenIsolines: location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles] # calculate the "actual" cost for each location estimate as first iteration + # NOTE: sometimes it cannot calculate the cost and returns None. Just assume isorange and stop the calculations there resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') - costs = [c['time'] for c in resp['one_to_many'][0][1:]] + costs = [(c['time'] or isorange) for c in resp['one_to_many'][0][1:]] #import pdb; pdb.set_trace() # iterate to refine the first solution, if needed for i in xrange(0, self.MAX_ITERS): + logging.debug('costs = %s' % costs) errors = [(cost - isorange) / float(isorange) for cost in costs] max_abs_error = max([abs(e) for e in errors]) if max_abs_error <= self.TOLERANCE: @@ -66,7 +77,7 @@ class MapzenIsolines: # and check "actual" costs again resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') - costs = [c['time'] for c in resp['one_to_many'][0][1:]] + costs = [(c['time'] or isorange) for c in resp['one_to_many'][0][1:]] return location_estimates From 6810dc0ff0eae489b155eb12cb8c648b962214f2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 5 Jul 2016 20:56:15 +0200 Subject: [PATCH 13/31] Code to glue together pg and python (WIP) --- server/extension/sql/80_isolines_helper.sql | 27 +++++++++++++++------ server/extension/sql/90_isochrone.sql | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 43ff444..bcb30a5 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_routing_isolines( +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines( username TEXT, orgname TEXT, isotype TEXT, @@ -65,6 +65,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_routing_isolines( options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ import json + from cartodb_services.mapzen import MatrixClient from cartodb_services.mapzen import MapzenIsolines from cartodb_services.metrics import QuotaService from cartodb_services.here.types import geo_polyline_to_multipolygon # TODO do we use the same types? @@ -82,24 +83,34 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_conf_str = plpy.execute("SELECT * FROM CDB_Conf_Getconf('mapzen_conf') AS mapzen_conf")[0]['mapzen_conf'] mapzen_conf = json.loads(mapzen_conf_str) - client = MapzenIsolines(mapzen_conf['routing']['api_key']) + + client = MatrixClient(mapzen_conf['matrix']['api_key']) + mapzen_isolines = MapzenIsolines(client) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - source_str = 'geo!%f,%f' % (lat, lon) + origin = {'lat': lat, 'lon': lon} else: - source_str = None + raise Exception('source is NULL') # -- TODO Support options properly + isolines = [] if isotype == 'isodistance': - resp = client.calculate_isodistance(source_str, mode, data_range) + # -- TODO implement + raise 'not implemented' + resp = mapzen_isolines.calculate_isodistance(origin, mode, data_range) elif isotype == 'isochrone': - resp = client.calculate_isochrone(source_str, mode, data_range) + for r in data_range: + isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) + isolines.append(isoline) - if resp: + return [] # -- TODO delete this + + # -- TODO rewrite this block + if isolines: result = [] - for isoline in resp: + for isoline in isolines: data_range_n = isoline['range'] polyline = isoline['geom'] multipolygon = geo_polyline_to_multipolygon(polyline) diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index a75ec67..83af69b 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -34,7 +34,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ #if user_isolines_config.google_services_user: # plpy.error('This service is not available for google service users.') - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) isolines = [] for element in result: From eb906fae352e4d20c2a062f7739ac35e56493019 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 11:20:39 +0200 Subject: [PATCH 14/31] Convert to multipolygon and return isolines --- server/extension/sql/80_isolines_helper.sql | 34 ++++++++++----------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index bcb30a5..98b5091 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ # plpy.error('You have reached the limit of your quota') try: - # TODO: encapsulate or refactor this ugly code + # --TODO: encapsulate or refactor this ugly code mapzen_conf_str = plpy.execute("SELECT * FROM CDB_Conf_Getconf('mapzen_conf') AS mapzen_conf")[0]['mapzen_conf'] mapzen_conf = json.loads(mapzen_conf_str) @@ -95,7 +95,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('source is NULL') # -- TODO Support options properly - isolines = [] + isolines = {} if isotype == 'isodistance': # -- TODO implement raise 'not implemented' @@ -103,24 +103,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ elif isotype == 'isochrone': for r in data_range: isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) - isolines.append(isoline) + isolines[r] = (isoline) - return [] # -- TODO delete this + result = [] + for r in data_range: - # -- TODO rewrite this block - if isolines: - result = [] - for isoline in isolines: - data_range_n = isoline['range'] - polyline = isoline['geom'] - multipolygon = geo_polyline_to_multipolygon(polyline) - result.append([source, data_range_n, multipolygon]) - #quota_service.increment_success_service_use() - #quota_service.increment_isolines_service_use(len(resp)) - return result - else: - #quota_service.increment_empty_service_use() - return [] + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + + result.append([source, r, multipolygon]) + # --TODO take care of this quota/usage stuff + #quota_service.increment_success_service_use() + #quota_service.increment_isolines_service_use(len(resp)) + return result except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() From 3dacb43a9a02dfa7f0e93e2d0040d8ec168d046e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 12:10:43 +0200 Subject: [PATCH 15/31] Add cdb_mapzen_isochrone to client On behalf of @iriberri. --- client/renderer/interface.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index b2917ca..0f3eb6c 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -103,6 +103,16 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_mapzen_isochrone + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_route_point_to_point return_type: cdb_dataservices_client.simple_route multi_field: true From cdcac2dc1f78922fee4bdf365ca643c90efec397 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 12:18:52 +0200 Subject: [PATCH 16/31] Fix typo in test case --- server/lib/python/cartodb_services/test/test_mapzengeocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py index 0c7be54..7eee6c0 100644 --- a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -11,7 +11,7 @@ requests_mock.Mocker.TEST_PREFIX = 'test_' @requests_mock.Mocker() -class GoogleGeocoderTestCase(unittest.TestCase): +class MapzenGeocoderTestCase(unittest.TestCase): MAPZEN_GEOCODER_URL = 'https://search.mapzen.com/v1/search' EMPTY_RESPONSE = """{ From a046d3ce977cec08b7106fc266325f0c99d2700f Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Wed, 6 Jul 2016 12:40:31 +0200 Subject: [PATCH 17/31] Add Mapzen Matrix to config and metrics services --- server/extension/sql/15_config_helper.sql | 14 ++++++ server/extension/sql/80_isolines_helper.sql | 26 +++++------ server/extension/sql/90_isochrone.sql | 6 +-- .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 45 +++++++++++++++++++ .../cartodb_services/metrics/quota.py | 3 ++ 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 3d6fd72..118bbec 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -26,6 +26,20 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenIsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_isolines_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 98b5091..874d8d1 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -71,20 +71,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.here.types import geo_polyline_to_multipolygon # TODO do we use the same types? redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] # -- Check the quota - #quota_service = QuotaService(user_isolines_routing_config, redis_conn) - #if not quota_service.check_user_quota(): - # plpy.error('You have reached the limit of your quota') + quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') try: - # --TODO: encapsulate or refactor this ugly code - mapzen_conf_str = plpy.execute("SELECT * FROM CDB_Conf_Getconf('mapzen_conf') AS mapzen_conf")[0]['mapzen_conf'] - mapzen_conf = json.loads(mapzen_conf_str) - - - client = MatrixClient(mapzen_conf['matrix']['api_key']) + client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key) mapzen_isolines = MapzenIsolines(client) if source: @@ -115,19 +110,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ multipolygon = plpy.execute(sql, 1)[0]['geom'] result.append([source, r, multipolygon]) - # --TODO take care of this quota/usage stuff - #quota_service.increment_success_service_use() - #quota_service.increment_isolines_service_use(len(resp)) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - #quota_service.increment_failed_service_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) plpy.debug(traceback.format_tb(traceback_)) raise e #plpy.error(error_msg) finally: - pass - #quota_service.increment_total_service_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 83af69b..8e24c94 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -26,11 +26,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] type = 'isochrone' - # if we were to add a config check, it'll go here + # If we were to add a config check, it'll go here #if user_isolines_config.google_services_user: # plpy.error('This service is not available for google service users.') diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 9bba57c..78d09e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig +from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 340e5f6..e4d6597 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -241,6 +241,41 @@ class IsolinesRoutingConfig(ServiceConfig): return self._geocoder_type == self.GOOGLE_GEOCODER +class MapzenIsolinesRoutingConfig(ServiceConfig): + + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn, + username, orgname) + try: + self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key + self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + self._cost_per_hit = 0 + except Exception as e: + raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e)) + + @property + def service_type(self): + return 'mapzen_isolines' + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return False + + @property + def period_end_date(self): + return self._period_end_date + + @property + def mapzen_matrix_api_key(self): + return self._mapzen_matrix_api_key + class InternalGeocoderConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -438,6 +473,8 @@ class ServicesDBConfig: raise ConfigException('Mapzen configuration missing') else: mapzen_conf = json.loads(mapzen_conf_json) + self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] + self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] @@ -492,6 +529,14 @@ class ServicesDBConfig: def heremaps_geocoder_cost_per_hit(self): return self._heremaps_geocoder_cost_per_hit + @property + def mapzen_matrix_api_key(self): + return self._mapzen_matrix_api_key + + @property + def mapzen_matrix_monthly_quota(self): + return self._mapzen_matrix_quota + @property def mapzen_routing_api_key(self): return self._mapzen_routing_api_key diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index c627769..4496e3c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -70,6 +70,9 @@ class QuotaChecker: elif re.match('here_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('mapzen_isolines', + self._user_service_config.service_type) is not None: + return self.__check_isolines_quota() elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() From 2147d190a12b1791249c070db83ccd728a21ea46 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 13:13:10 +0200 Subject: [PATCH 18/31] Unit test for the mapzen isolines --- .../python/cartodb_services/test/__init__.py | 0 .../test/test_mapzenisolines.py | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/__init__.py create mode 100644 server/lib/python/cartodb_services/test/test_mapzenisolines.py diff --git a/server/lib/python/cartodb_services/test/__init__.py b/server/lib/python/cartodb_services/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/test_mapzenisolines.py b/server/lib/python/cartodb_services/test/test_mapzenisolines.py new file mode 100644 index 0000000..69f5597 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_mapzenisolines.py @@ -0,0 +1,76 @@ +import unittest +from cartodb_services.mapzen import MapzenIsolines +from math import radians, cos, sin, asin, sqrt + +""" +This file is basically a sanity test on the algorithm. + + +It uses a mocked client, which returns the cost based on a very simple model: +just proportional to the distance from origin to the target point. +""" + +class MatrixClientMock(): + + def __init__(self, speed): + """ + Sets up the mock with a speed in km/h + """ + self._speed = speed + + def one_to_many(self, locations, costing): + origin = locations[0] + distances = [self._distance(origin, l) for l in locations] + response = { + 'one_to_many': [ + [ + { + 'distance': distances[i] * self._speed, + 'time': distances[i] / self._speed * 3600, + 'to_index': i, + 'from_index': 0 + } + for i in xrange(0, len(distances)) + ] + ], + 'units': 'km', + 'locations': [ + locations + ] + } + return response + + def _distance(self, a, b): + """ + Calculate the great circle distance between two points + on the earth (specified in decimal degrees) + http://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing-and-distance-between-two-gps-points + + Returns: + distance in meters + """ + + # convert decimal degrees to radians + lon1, lat1, lon2, lat2 = map(radians, [a['lon'], a['lat'], b['lon'], b['lat']]) + + # haversine formula + dlon = lon2 - lon1 + dlat = lat2 - lat1 + a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2 + c = 2 * asin(sqrt(a)) + r = 6371 # Radius of earth in kilometers. Use 3956 for miles + return c * r + + +class MapzenIsolinesTestCase(unittest.TestCase): + + def setUp(self): + speed = 6 # in km/h + matrix_client = MatrixClientMock(speed) + self.mapzen_isolines = MapzenIsolines(matrix_client) + + def test_calculate_isochrone(self): + origin = {"lat":40.744014,"lon":-73.990508} + transport_mode = 'walk' + isorange = 10 * 60 # 10 minutes + solution = self.mapzen_isolines.calculate_isochrone(origin, transport_mode, isorange) From ff4eb5b3480b6d6d8fd6dedbf6c57ccd2dd1149c Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Wed, 6 Jul 2016 14:21:32 +0200 Subject: [PATCH 19/31] Mock mapzen matrix config --- server/lib/python/cartodb_services/test/test_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index b0ffaf0..fe6fd8f 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -65,7 +65,7 @@ def _plpy_execute_side_effect(*args, **kwargs): if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}}'}] + return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": From 7ddb3da60d5567c3e52f27cfe65e187e411bbea9 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Wed, 6 Jul 2016 15:43:26 +0200 Subject: [PATCH 20/31] Remove useless cost_per_hit line --- .../python/cartodb_services/cartodb_services/metrics/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index e4d6597..a5b80fa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -252,7 +252,6 @@ class MapzenIsolinesRoutingConfig(ServiceConfig): self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) - self._cost_per_hit = 0 except Exception as e: raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e)) From 6c4829df015278fced84f86fda0c1fd5d0a63d1e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 16:04:41 +0200 Subject: [PATCH 21/31] Small refactor for sanity --- .../cartodb_services/mapzen/isolines.py | 22 +++++++++---------- .../test/test_mapzenisolines.py | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 31b7c01..9a75075 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -31,6 +31,7 @@ class MapzenIsolines: # NOTE: not for production #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log') + #logging.basicConfig(level=logging.DEBUG) logging.debug('origin = %s' % origin) logging.debug('transport_mode = %s' % transport_mode) logging.debug('isorange = %d' % isorange) @@ -50,15 +51,15 @@ class MapzenIsolines: rmin = [0.0] * self.NUMBER_OF_ANGLES location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles] - # calculate the "actual" cost for each location estimate as first iteration - # NOTE: sometimes it cannot calculate the cost and returns None. Just assume isorange and stop the calculations there - resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') - costs = [(c['time'] or isorange) for c in resp['one_to_many'][0][1:]] - #import pdb; pdb.set_trace() - - # iterate to refine the first solution, if needed + # Iterate to refine the first solution for i in xrange(0, self.MAX_ITERS): - logging.debug('costs = %s' % costs) + # Calculate the "actual" cost for each location estimate. + # NOTE: sometimes it cannot calculate the cost and returns None. + # Just assume isorange and stop the calculations there + response = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') + costs = [(c['time'] or isorange) for c in response['one_to_many'][0][1:]] + logging.debug('i = %d, costs = %s' % (i, costs)) + errors = [(cost - isorange) / float(isorange) for cost in costs] max_abs_error = max([abs(e) for e in errors]) if max_abs_error <= self.TOLERANCE: @@ -73,11 +74,8 @@ class MapzenIsolines: rmax[j] = (rmax[j] + rmin[j]) / 2.0 else: rmin[j] = (rmax[j] + rmin[j]) / 2.0 - location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) - # and check "actual" costs again - resp = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') - costs = [(c['time'] or isorange) for c in resp['one_to_many'][0][1:]] + location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) return location_estimates diff --git a/server/lib/python/cartodb_services/test/test_mapzenisolines.py b/server/lib/python/cartodb_services/test/test_mapzenisolines.py index 69f5597..4fb4e96 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenisolines.py +++ b/server/lib/python/cartodb_services/test/test_mapzenisolines.py @@ -65,7 +65,7 @@ class MatrixClientMock(): class MapzenIsolinesTestCase(unittest.TestCase): def setUp(self): - speed = 6 # in km/h + speed = 4 # in km/h matrix_client = MatrixClientMock(speed) self.mapzen_isolines = MapzenIsolines(matrix_client) From 075edf0e0d595d44deb3d485ffc7ef18cbf326f7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 18:01:33 +0200 Subject: [PATCH 22/31] More precission for earth's radius --- .../python/cartodb_services/cartodb_services/mapzen/isolines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 9a75075..95294ad 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -7,7 +7,7 @@ class MapzenIsolines: MAX_ITERS = 5 TOLERANCE = 0.1 - EARTH_RADIUS_METERS = 6371000 + EARTH_RADIUS_METERS = 6367444 def __init__(self, matrix_client): self._matrix_client = matrix_client From 6d888a7a625fcf5b9b24761afdb973cef4d8147b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 18:03:20 +0200 Subject: [PATCH 23/31] Fix for points getting None cost Sometimes there's no route information for the point in a particular angle we're interested in. In this case it is better to use more points/angles and discard the ones we're not interested in. --- .../cartodb_services/mapzen/isolines.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 95294ad..a78cf4a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -3,7 +3,7 @@ import logging class MapzenIsolines: - NUMBER_OF_ANGLES = 12 + NUMBER_OF_ANGLES = 24 MAX_ITERS = 5 TOLERANCE = 0.1 @@ -77,7 +77,13 @@ class MapzenIsolines: location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0) - return location_estimates + # delete points that got None + location_estimates_filtered = [] + for i, c in enumerate(costs): + if c <> isorange: + location_estimates_filtered.append(location_estimates[i]) + + return location_estimates_filtered From 54221fa67185892d3421304c6ed0406dc3873b19 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 18:05:51 +0200 Subject: [PATCH 24/31] Add transport mode car --- .../cartodb_services/mapzen/isolines.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index a78cf4a..63aa2b3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -36,17 +36,21 @@ class MapzenIsolines: logging.debug('transport_mode = %s' % transport_mode) logging.debug('isorange = %d' % isorange) + if transport_mode == 'walk': + upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed + costing_model = 'pedestrian' + elif transport_mode == 'car': + upper_rmax = 41.67 * isorange # assuming 140km/h max speed + costing_model = 'auto' + else: + raise NotImplementedError('car and walk are the only supported modes for the moment') - if transport_mode != 'walk': - # TODO move this restriction to the appropriate place - raise NotImplementedError('walk is the only supported mode for the moment') # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE # Initial setup angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles - upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed rmax = [upper_rmax] * self.NUMBER_OF_ANGLES rmin = [0.0] * self.NUMBER_OF_ANGLES location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles] @@ -56,8 +60,10 @@ class MapzenIsolines: # Calculate the "actual" cost for each location estimate. # NOTE: sometimes it cannot calculate the cost and returns None. # Just assume isorange and stop the calculations there - response = self._matrix_client.one_to_many([origin] + location_estimates, 'pedestrian') + + response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model) costs = [(c['time'] or isorange) for c in response['one_to_many'][0][1:]] + logging.debug('i = %d, costs = %s' % (i, costs)) errors = [(cost - isorange) / float(isorange) for cost in costs] From 523eda2cc797dbe97f25b170071becd78a4d1555 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 18:43:09 +0200 Subject: [PATCH 25/31] Generalize calculate_isochrone to calculate_isoline --- .../cartodb_services/mapzen/isolines.py | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 63aa2b3..6ad7464 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -27,25 +27,46 @@ class MapzenIsolines: Returns: Array of {lon: x, lat: y} as a representation of the isoline """ - def calculate_isochrone(self, origin, transport_mode, isorange): + def calculate_isochrone(self, origin, transport_mode, time_range): + if transport_mode == 'walk': + max_speed = 3.3333333 # In m/s, assuming 12km/h walking speed + costing_model = 'pedestrian' + elif transport_mode == 'car': + max_speed = 41.67 # In m/s, assuming 140km/h max speed + costing_model = 'auto' + else: + raise NotImplementedError('car and walk are the only supported modes for the moment') + + upper_rmax = max_speed * time_range # an upper bound for the radius + + return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'time') + + """Get an isoline using mapzen API. + + The implementation tries to sick close to the SQL API: + cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline + + But this calculates just one isoline. + + Args: + origin dict containing {lat: y, lon: x} + costing_model string "auto" or "pedestrian" + isorange int Range of the isoline in seconds + upper_rmax float An upper bound for the binary search + cost_variable string Variable to optimize "time" or "distance" + + Returns: + Array of {lon: x, lat: y} as a representation of the isoline + """ + def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable): # NOTE: not for production #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log') #logging.basicConfig(level=logging.DEBUG) logging.debug('origin = %s' % origin) - logging.debug('transport_mode = %s' % transport_mode) + logging.debug('costing_model = %s' % costing_model) logging.debug('isorange = %d' % isorange) - if transport_mode == 'walk': - upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed - costing_model = 'pedestrian' - elif transport_mode == 'car': - upper_rmax = 41.67 * isorange # assuming 140km/h max speed - costing_model = 'auto' - else: - raise NotImplementedError('car and walk are the only supported modes for the moment') - - # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE @@ -62,7 +83,7 @@ class MapzenIsolines: # Just assume isorange and stop the calculations there response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model) - costs = [(c['time'] or isorange) for c in response['one_to_many'][0][1:]] + costs = [(c[cost_variable] or isorange) for c in response['one_to_many'][0][1:]] logging.debug('i = %d, costs = %s' % (i, costs)) From 230112b7e5657ccd0d2e540625a63d0682fb70ef Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 19:20:21 +0200 Subject: [PATCH 26/31] Add calculate_isodistance function --- .../cartodb_services/mapzen/isolines.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index 6ad7464..cd521c0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -41,6 +41,29 @@ class MapzenIsolines: return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'time') + """Get an isodistance using mapzen API. + + Args: + origin dict containing {lat: y, lon: x} + transport_mode string, for the moment just "car" or "walk" + isorange int range of the isoline in seconds + + Returns: + Array of {lon: x, lat: y} as a representation of the isoline + """ + def calculate_isodistance(self, origin, transport_mode, distance_range): + if transport_mode == 'walk': + costing_model = 'pedestrian' + elif transport_mode == 'car': + costing_model = 'auto' + else: + raise NotImplementedError('car and walk are the only supported modes for the moment') + + upper_rmax = distance_range # an upper bound for the radius, going in a straight line + + return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'distance', 1000.0) + + """Get an isoline using mapzen API. The implementation tries to sick close to the SQL API: @@ -54,11 +77,12 @@ class MapzenIsolines: isorange int Range of the isoline in seconds upper_rmax float An upper bound for the binary search cost_variable string Variable to optimize "time" or "distance" + unit_factor float A factor to adapt units of isorange (meters) and units of distance (km) Returns: Array of {lon: x, lat: y} as a representation of the isoline """ - def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable): + def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0): # NOTE: not for production #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log') @@ -83,7 +107,7 @@ class MapzenIsolines: # Just assume isorange and stop the calculations there response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model) - costs = [(c[cost_variable] or isorange) for c in response['one_to_many'][0][1:]] + costs = [(c[cost_variable]*unit_factor or isorange) for c in response['one_to_many'][0][1:]] logging.debug('i = %d, costs = %s' % (i, costs)) From 99798f26180f7a3c4220e4ff5151372cffc4c230 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 19:40:40 +0200 Subject: [PATCH 27/31] Integrate isodistance into SQL API --- client/renderer/interface.yaml | 10 ++++++++++ server/extension/sql/80_isolines_helper.sql | 7 +++---- server/extension/sql/85_isodistance.sql | 15 +++++++++++++++ server/extension/sql/90_isochrone.sql | 11 +---------- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 0f3eb6c..10b4ef2 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -113,6 +113,16 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } +- name: cdb_mapzen_distance + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - name: cdb_route_point_to_point return_type: cdb_dataservices_client.simple_route multi_field: true diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 874d8d1..f04e62c 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -68,7 +68,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.mapzen import MatrixClient from cartodb_services.mapzen import MapzenIsolines from cartodb_services.metrics import QuotaService - from cartodb_services.here.types import geo_polyline_to_multipolygon # TODO do we use the same types? redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] @@ -92,9 +91,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ # -- TODO Support options properly isolines = {} if isotype == 'isodistance': - # -- TODO implement - raise 'not implemented' - resp = mapzen_isolines.calculate_isodistance(origin, mode, data_range) + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = (isoline) elif isotype == 'isochrone': for r in data_range: isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 3c3ac0e..29beddb 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -19,3 +19,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ return isolines $$ LANGUAGE plpythonu; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 8e24c94..409cf39 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -30,17 +30,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] type = 'isochrone' - # If we were to add a config check, it'll go here - #if user_isolines_config.google_services_user: - # plpy.error('This service is not available for google service users.') - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) - isolines = [] - for element in result: - isoline = element['isoline'] - isoline = isoline.translate(None, "()").split(',') - isolines.append(isoline) - return isolines + return result $$ LANGUAGE plpythonu; From bcc6bc35d3973ccfdaa80c7859cdddfd63a4a085 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 19:48:20 +0200 Subject: [PATCH 28/31] Fix None*unit_factor error Also make the code more explicit about what happens when getting cost == None. --- .../cartodb_services/cartodb_services/mapzen/isolines.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index cd521c0..d686407 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -107,7 +107,12 @@ class MapzenIsolines: # Just assume isorange and stop the calculations there response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model) - costs = [(c[cost_variable]*unit_factor or isorange) for c in response['one_to_many'][0][1:]] + costs = [None] * self.NUMBER_OF_ANGLES + for idx, c in enumerate(response['one_to_many'][0][1:]): + if c[cost_variable]: + costs[idx] = c[cost_variable]*unit_factor + else: + costs[idx] = isorange logging.debug('i = %d, costs = %s' % (i, costs)) From 9b7a2d491f2b3ada4a065f139b64f389ed488c1f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 19:58:04 +0200 Subject: [PATCH 29/31] Fix bug adapting types passing through plpython --- server/extension/sql/85_isodistance.sql | 7 ++++++- server/extension/sql/90_isochrone.sql | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 29beddb..6349015 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -31,6 +31,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) - return result + return isolines $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 409cf39..11f11cb 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -32,6 +32,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') #--TODO what is this for? + isolines.append(isoline) - return result + return isolines $$ LANGUAGE plpythonu; From 9c87762b8bfeaf22f7a1c39c6e51a41a11c088f8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 20:03:17 +0200 Subject: [PATCH 30/31] Fix typo in client interface --- client/renderer/interface.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 10b4ef2..97c7675 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -113,7 +113,7 @@ - { name: range, type: "integer[]" } - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } -- name: cdb_mapzen_distance +- name: cdb_mapzen_isodistance return_type: SETOF cdb_dataservices_client.isoline multi_row: true multi_field: true From f5d51da673e1facba35c5bd31a0f26e97d23e827 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Jul 2016 20:05:37 +0200 Subject: [PATCH 31/31] Fix another typo (hello Carla!!) This feature is dedicated to you. Keep rocking. --- .../python/cartodb_services/cartodb_services/mapzen/isolines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index d686407..de75459 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -61,7 +61,7 @@ class MapzenIsolines: upper_rmax = distance_range # an upper bound for the radius, going in a straight line - return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'distance', 1000.0) + return self.calculate_isoline(origin, costing_model, distance_range, upper_rmax, 'distance', 1000.0) """Get an isoline using mapzen API.