cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #622 from CartoDB/feature/ch115970/support-new-here-api-keys

Browse Source 
Feature/ch115970/support new here api keys
This commit is contained in:
Miguel Moncada Isla 2021-05-26 10:36:39 +02:00 committed by GitHub
commit 7862e70bab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 3270 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -132,6 +132,15 @@ SELECT CDB_Conf_SetConf(
);
```
Use HERE apikeys
```sql
SELECT CDB_Conf_SetConf(
'heremaps_conf',
'{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true, "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true}}'
);
```
#### Mapzen configuration
```sql

2
server/extension/Makefile
View File

@ -1,7 +1,7 @@
# Makefile to generate the extension out of separate sql source files.
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
EXTENSION = cdb_dataservices_server
EXTVERSION = 0.40.0
EXTVERSION = 0.40.1
# The new version to be generated from templates
SED ?= sed

110
server/extension/cdb_dataservices_server--0.40.0--0.40.1.sql Normal file
View File

@ -0,0 +1,110 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.40.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.tools import QuotaExceededException
from cartodb_services.here import get_geocoder
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
try:
service_manager.assert_within_limits()
geocoder = get_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
service_manager.quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
service_manager.quota_service.increment_empty_service_use()
return None
except QuotaExceededException as qe:
service_manager.quota_service.increment_failed_service_use()
return None
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using here maps')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services import run_street_point_geocoder
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.here import get_bulk_geocoder
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
geocoder = get_bulk_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.here import get_routing_isoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
client = get_routing_isoline(app_id=user_isolines_routing_config.heremaps_app_id or None,
app_code=user_isolines_routing_config.heremaps_app_code or None, logger=logger,
apikey=user_isolines_routing_config.heremaps_apikey or None,
use_apikey=user_isolines_routing_config.heremaps_use_apikey or False)
client_api_version = client.get_api_version()
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
if client_api_version == 7:
source_str = 'geo!%f,%f' % (lat, lon)
else:
source_str = '%f,%f' % (lat, lon)
else:
source_str = None
if type == 'isodistance':
resp = client.calculate_isodistance(source_str, mode, data_range, options)
elif type == 'isochrone':
resp = client.calculate_isochrone(source_str, mode, data_range, options)
if resp:
result = []
for isoline in resp:
data_range_n = isoline['range']
polyline = isoline['geom']
multipolygon = geo_polyline_to_multipolygon(polyline)
result.append([source, data_range_n, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(resp))
return result
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

105
server/extension/cdb_dataservices_server--0.40.1--0.40.0.sql Normal file
View File

@ -0,0 +1,105 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.40.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.tools import QuotaExceededException
from cartodb_services.here import HereMapsGeocoder
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
try:
service_manager.assert_within_limits()
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
service_manager.quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
service_manager.quota_service.increment_empty_service_use()
return None
except QuotaExceededException as qe:
service_manager.quota_service.increment_failed_service_use()
return None
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using here maps')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services import run_street_point_geocoder
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.here import HereMapsBulkGeocoder
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
user_isolines_routing_config.heremaps_app_code, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
source_str = 'geo!%f,%f' % (lat, lon)
else:
source_str = None
if type == 'isodistance':
resp = client.calculate_isodistance(source_str, mode, data_range, options)
elif type == 'isochrone':
resp = client.calculate_isochrone(source_str, mode, data_range, options)
if resp:
result = []
for isoline in resp:
data_range_n = isoline['range']
polyline = isoline['geom']
multipolygon = geo_polyline_to_multipolygon(polyline)
result.append([source, data_range_n, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(resp))
return result
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

2419
server/extension/cdb_dataservices_server--0.40.1.sql Normal file
View File

File diff suppressed because it is too large Load Diff

0
server/extension/cdb_dataservices_server--0.39.3--0.40.0.sql → server/extension/old_versions/cdb_dataservices_server--0.39.3--0.40.0.sql
View File

0
server/extension/cdb_dataservices_server--0.40.0--0.39.3.sql → server/extension/old_versions/cdb_dataservices_server--0.40.0--0.39.3.sql
View File

0
server/extension/cdb_dataservices_server--0.40.0.sql → server/extension/old_versions/cdb_dataservices_server--0.40.0.sql
View File

4
server/extension/sql/20_geocode_street.sql
View File

@ -125,14 +125,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
RETURNS Geometry AS $$
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.tools import QuotaExceededException
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here import get_geocoder
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
try:
service_manager.assert_within_limits()
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
geocoder = get_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
service_manager.quota_service.increment_success_service_use()

4
server/extension/sql/21_bulk_geocode_street.sql
View File

@ -57,10 +57,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_st
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services import run_street_point_geocoder
from cartodb_services.tools import LegacyServiceManager
from cartodb_services.here import HereMapsBulkGeocoder
from cartodb_services.here import get_bulk_geocoder
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
geocoder = get_bulk_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

15
server/extension/sql/80_isolines_helper.sql
View File

@ -3,7 +3,7 @@ CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326),
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.here import get_routing_isoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger,LoggerConfig
@ -20,13 +20,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
raise Exception('You have reached the limit of your quota')
try:
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
user_isolines_routing_config.heremaps_app_code, logger)
client = get_routing_isoline(app_id=user_isolines_routing_config.heremaps_app_id or None,
app_code=user_isolines_routing_config.heremaps_app_code or None, logger=logger,
apikey=user_isolines_routing_config.heremaps_apikey or None,
use_apikey=user_isolines_routing_config.heremaps_use_apikey or False)
client_api_version = client.get_api_version()
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
source_str = 'geo!%f,%f' % (lat, lon)
if client_api_version == 7:
source_str = 'geo!%f,%f' % (lat, lon)
else:
source_str = '%f,%f' % (lat, lon)
else:
source_str = None

5
server/lib/python/cartodb_services/cartodb_services/here/__init__.py
View File

@ -1,3 +1,4 @@
from cartodb_services.here.geocoder import HereMapsGeocoder
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
from cartodb_services.here.geocoder import HereMapsGeocoder, HereMapsGeocoderV7
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder, HereMapsBulkGeocoderV7
from cartodb_services.here.service_factory import get_geocoder, get_bulk_geocoder, get_routing_isoline
from cartodb_services.here.routing import HereMapsRoutingIsoline

141
server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py
View File

@ -7,7 +7,7 @@ import xml.etree.ElementTree as ET
from collections import namedtuple
from requests.adapters import HTTPAdapter
from cartodb_services import StreetPointBulkGeocoder
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here import HereMapsGeocoder, HereMapsGeocoderV7
from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED
from cartodb_services.metrics import Traceable
from cartodb_services.tools.exceptions import ServiceException
@ -26,7 +26,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults)
HereMapsGeocoder.__init__(self, app_id=app_id, app_code=app_code, logger=logger, service_params=service_params, maxresults=maxresults)
self.session = requests.Session()
self.session.mount(self.BATCH_URL,
HTTPAdapter(max_retries=self.max_retries))
@ -153,3 +153,140 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
return results
class HereMapsBulkGeocoderV7(HereMapsGeocoderV7, StreetPointBulkGeocoder):
MAX_BATCH_SIZE = 1000000 # From the docs
MIN_BATCHED_SEARCH = 100 # Under this, serial will be used
BATCH_URL = 'https://batch.geocoder.ls.hereapi.com/6.2/jobs'
# https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
MAX_STALLED_RETRIES = 100
BATCH_RETRY_SLEEP_S = 5
JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
def __init__(self, apikey, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
HereMapsGeocoderV7.__init__(self, apikey=apikey, logger=logger, service_params=service_params, limit=maxresults)
self.apikey = apikey
self.session = requests.Session()
self.session.mount(self.BATCH_URL,
HTTPAdapter(max_retries=self.max_retries))
self.credentials_params = {
'apikey': self.apikey
}
def _should_use_batch(self, searches):
return len(searches) >= self.MIN_BATCHED_SEARCH
def _serial_geocode(self, searches):
results = []
for search in searches:
(search_id, address, city, state, country) = search
try:
result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
except Exception as e:
self._logger.error("Error geocoding", e)
result = geocoder_error_response("Error geocoding")
results.append((search_id, result[0], result[1]))
return results
def _batch_geocode(self, searches):
request_id = self._send_batch(self._searches_to_csv(searches))
last_processed = 0
stalled_retries = 0
# https://developer.here.com/documentation/batch-geocoder/topics/job-status.html
while True:
job_info = self._job_status(request_id)
if job_info.processed_count == last_processed:
stalled_retries += 1
if stalled_retries > self.MAX_STALLED_RETRIES:
raise Exception('Too many retries for job {}'.format(request_id))
else:
stalled_retries = 0
last_processed = job_info.processed_count
if job_info.status in self.JOB_FINAL_STATES:
break
else:
time.sleep(self.BATCH_RETRY_SLEEP_S)
results = self._download_results(request_id)
return results
def _searches_to_csv(self, searches):
queue = cStringIO.StringIO()
writer = csv.writer(queue, delimiter='|')
writer.writerow(['recId', 'searchText', 'country'])
for search in searches:
fields = [search.address, search.city, search.state]
search_text = ', '.join(filter(None, fields))
row = [s.encode("utf-8") if s else ''
for s in [str(search.id), search_text, search.country]]
writer.writerow(row)
return queue.getvalue()
def _send_batch(self, data):
cols = 'displayLatitude,displayLongitude,' + ','.join(self.META_COLS)
request_params = self.credentials_params.copy()
request_params.update({
'gen': 8,
'action': 'run',
'header': 'true',
'inDelim': '|',
'outDelim': '|',
'outCols': cols,
'outputcombined': 'true'
})
response = self.session.post(self.BATCH_URL, data=data,
params=request_params,
timeout=(self.connect_timeout, self.read_timeout))
if response.status_code == 200:
root = ET.fromstring(response.text)
return root.find('./Response/MetaInfo/RequestId').text
else:
raise ServiceException("Error sending HERE batch", response)
def _job_status(self, request_id):
polling_params = self.credentials_params.copy()
polling_params.update({'action': 'status'})
polling_r = self.session.get("{}/{}".format(self.BATCH_URL, request_id),
params=polling_params,
timeout=(self.connect_timeout, self.read_timeout))
polling_root = ET.fromstring(polling_r.text)
return HereJobStatus(
total_count=int(polling_root.find('./Response/TotalCount').text),
processed_count=int(polling_root.find('./Response/ProcessedCount').text),
status=polling_root.find('./Response/Status').text)
def _download_results(self, job_id):
result_r = self.session.get("{}/{}/all".format(self.BATCH_URL, job_id),
params=self.credentials_params,
timeout=(self.connect_timeout, self.read_timeout))
root_zip = zipfile.ZipFile(io.BytesIO(result_r.content))
results = []
for name in root_zip.namelist():
if name.endswith('_out.txt'):
reader = csv.DictReader(root_zip.open(name), delimiter='|')
for row in reader:
if row['SeqNumber'] == '1': # First per requested data
precision = self.PRECISION_BY_MATCH_TYPE.get(
row.get('matchType'), PRECISION_INTERPOLATED)
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None)
results.append((row['recId'],
[row['displayLongitude'], row['displayLatitude']],
geocoder_metadata(
float(row['relevance']),
precision,
[match_type] if match_type else []
)))
elif row['matchLevel'] == 'NOMATCH':
results.append((row['recId'], [], {}))
elif row['matchLevel'] == 'FAILED':
results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
return results

185
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@ -8,9 +8,10 @@ from requests.adapters import HTTPAdapter
from exceptions import *
from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE
from cartodb_services.metrics import Traceable
from cartodb_services.tools.country import country_to_iso3
class HereMapsGeocoder(Traceable):
'A Here Maps Geocoder wrapper for python'
'A Here Maps v6 Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
@ -30,7 +31,7 @@ class HereMapsGeocoder(Traceable):
'searchtext',
'state',
'street'
]
]
ADMITTED_PARAMS = [
'additionaldata',
@ -50,7 +51,7 @@ class HereMapsGeocoder(Traceable):
'politicalview',
'prox',
'strictlanguagemode'
] + ADDRESS_PARAMS
] + ADDRESS_PARAMS
PRECISION_BY_MATCH_TYPE = {
'pointAddress': PRECISION_PRECISE,
@ -151,3 +152,181 @@ class HereMapsGeocoder(Traceable):
precision,
[match_type] if match_type else []
)
class HereMapsGeocoderV7(Traceable):
'A Here Maps Geocoder v7 wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocode.search.hereapi.com/v1/geocode'
DEFAULT_MAXRESULTS = 1
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
MAX_RETRIES=1
COUNTRY_ISO3_LENGTH = 3
ADDRESS_PARAMS = [
'city',
'country',
'county',
'district',
'housenumber',
'postalcode',
'searchtext',
'state',
'street'
]
QUALIFIED_PARAMS = [
'state',
'city',
'county',
'district',
'postalcode',
'housenumber',
'street'
]
PRECISION_BY_MATCH_TYPE = {
'PA': PRECISION_PRECISE,
'interpolated': PRECISION_INTERPOLATED
}
MATCH_TYPE_BY_MATCH_LEVEL = {
'landmark': 'point_of_interest',
'country': 'country',
'state': 'state',
'county': 'county',
'city': 'locality',
'district': 'district',
'street': 'street',
'intersection': 'intersection',
'houseNumber': 'street_number',
'postalCode': 'postal_code'
}
def __init__(self, apikey, logger, service_params=None, limit=DEFAULT_MAXRESULTS):
service_params = service_params or {}
self.apikey = apikey
self._logger = logger
self.limit = limit
self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL)
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
def geocode(self, **kwargs):
return self.geocode_meta(**kwargs)[0]
def geocode_meta(self, **kwargs):
params = {}
for key, value in kwargs.iteritems():
if value and value.strip():
params[key] = value
if not params:
return EMPTY_RESPONSE
return self._execute_geocode(params)
def _execute_geocode(self, params):
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
raise BadGeocodingParams(params)
try:
response = self._perform_request(params)
result = response['items'][0]
return [self._extract_lng_lat_from_result(result),
self._extract_metadata_from_result(result)]
except IndexError:
return EMPTY_RESPONSE
except KeyError as e:
self._logger.error('params: {}'.format(params), e)
raise MalformedResult()
def _get_v7param(self, param, reverse=False):
mapping = {
'city': 'city',
'country': 'in',
'county': 'county',
'district': 'district',
'housenumber': 'houseNumber',
'postalcode': 'postalCode',
'state': 'state',
'street': 'street',
'searchtext': 'q'
}
if reverse is True:
mapping = {v:k for k,v in mapping.items()}
return mapping.get(param)
def _is_iso3_country(self, country):
return len(country) == self.COUNTRY_ISO3_LENGTH
def _parse_country(self, country):
country_iso3 = country_to_iso3(country) or country
parsed_country = "countryCode:{0}".format(country_iso3) if (country_iso3 and self._is_iso3_country(country_iso3)) else None
return parsed_country
def _get_qq(self, params):
qualified_params = ['{0}={1}'.format(self._get_v7param(k), v) for k,v in params.items() if k in self.QUALIFIED_PARAMS]
qq = '&'.join(qualified_params) if qualified_params else None
return qq
def _parse_params(self, params):
country, q = (params.get(value) for value in ('country', 'searchtext'))
parsed_params = {'q': q}
qq = self._get_qq(params)
parsed_country = self._parse_country(country)
parsed_params.update({k:v for k, v in (('qq', qq), ('in', parsed_country)) if v is not None})
return parsed_params
def _perform_request(self, params):
request_params = {
'apikey': self.apikey,
'limit': self.limit,
}
parsed_params = self._parse_params(params)
request_params.update(parsed_params)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self.host, HTTPAdapter(max_retries=self.max_retries))
response = session.get(self.host, params=request_params,
timeout=(self.connect_timeout, self.read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:
self._logger.warning('Error 4xx trying to geocode street using HERE',
data={"response": response.json(), "params":
parsed_params})
return EMPTY_RESPONSE
else:
self._logger.error('Error trying to geocode street using HERE',
data={"response": response.json(), "params":
parsed_params})
raise Exception('Error trying to geocode street using Here')
def _extract_lng_lat_from_result(self, result):
position = result['position']
longitude = position['lng']
latitude = position['lat']
return [longitude, latitude]
def _extract_metadata_from_result(self, result):
# See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses
precision = self.PRECISION_BY_MATCH_TYPE.get(
result.get('houseNumberType'), PRECISION_INTERPOLATED)
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result.get('resultType'), None)
return geocoder_metadata(
result['scoring']['queryScore'],
precision,
[match_type] if match_type else []
)

202
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -1,5 +1,6 @@
import requests
import json
import flexpolyline as fp
from cartodb_services.here.exceptions import WrongParams
from requests.adapters import HTTPAdapter
@ -7,11 +8,12 @@ from cartodb_services.metrics import Traceable
class HereMapsRoutingIsoline(Traceable):
'A Here Maps Routing wrapper for python'
'A Here Maps Routing v7 wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
API_VERSION = 7
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
MAX_RETRIES = 1
@ -42,6 +44,9 @@ class HereMapsRoutingIsoline(Traceable):
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
self._url = "{0}{1}".format(base_url, isoline_path)
def get_api_version(self):
return self.API_VERSION
def calculate_isodistance(self, source, mode, data_range, options=[]):
return self.__calculate_isolines(source, mode, data_range, 'distance',
options)
@ -51,7 +56,8 @@ class HereMapsRoutingIsoline(Traceable):
options)
def __calculate_isolines(self, source, mode, data_range, range_type,
options=[]):
options=None):
options = options or []
parsed_options = self.__parse_options(options)
source_param = self.__parse_source_param(source, parsed_options)
mode_param = self.__parse_mode_param(mode, parsed_options)
@ -150,3 +156,195 @@ class HereMapsRoutingIsoline(Traceable):
mode_param = "{0};{1}".format(mode_param, mode_feature)
return {'mode': mode_param}
class HereMapsRoutingIsolineV8(Traceable):
'A Here Maps Routing v8 wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.router.hereapi.com/v8/isolines'
API_VERSION = 8
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
MAX_RETRIES = 1
QUALITY_PARAM_V7 = 'quality'
DEFAULT_OPTIMIZEFOR = 'quality'
DEFAULT_ROUTINGMODE = 'short'
HERE_WALK_MODE = "pedestrian"
HERE_CAR_MODE = "car"
ACCEPTED_MODES = {
"walk": HERE_WALK_MODE,
"car": HERE_CAR_MODE
}
OPTIONAL_PARAMS = [
'departure',
'arrival',
'maxpoints',
'quality'
]
def __init__(self, apikey, logger, service_params=None):
service_params = service_params or {}
self._apikey = apikey
self._logger = logger
self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
def get_api_version(self):
return self.API_VERSION
def calculate_isodistance(self, source, mode, data_range, options=None):
options = options or []
return self.__calculate_isolines(source, mode, data_range, 'distance',
options)
def calculate_isochrone(self, source, mode, data_range, options=None):
options = options or []
return self.__calculate_isolines(source, mode, data_range, 'time',
options)
def __calculate_isolines(self, source, mode, data_range, range_type,
options=None):
options = options or []
parsed_options = self.__parse_options(options)
source_param = self.__parse_source_param(source, parsed_options)
mode_params = self.__get_mode_params(mode, parsed_options)
request_params = self.__parse_request_parameters(source_param,
mode_params,
data_range,
range_type,
parsed_options)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.max_retries))
response = requests.get(self._url, params=request_params,
timeout=(self.connect_timeout, self.read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text)
elif response.status_code == requests.codes.bad_request:
return []
else:
self._logger.error('Error trying to calculate HERE isolines',
data={"response_status": response.status_code,
"response_reason": response.reason,
"response_content": response.text,
"reponse_url": response.url,
"response_headers": response.headers,
"source": source, "mode": mode,
"data_range": data_range,
"range_type": range_type,
"options": options})
raise Exception('Error trying to calculate HERE isolines')
def __parse_options(self, options):
return dict(option.split('=') for option in options)
def __get_v8_param(self, param, reverse=False):
mapping = {
'range': 'range[values]',
'rangetype': 'range[type]',
'mode': {
'type': 'routingmode',
'transportmodes': 'transportmode',
'trafficmode': None,
'feature': 'avoid[feature]'
},
'quality': 'optimizefor',
'maxpoints': 'shape[maxpoints]',
'start': 'origin',
'destination': 'destination',
'departure': 'departuretime',
'arrival': 'arrivaltime'
}
if reverse is True:
mapping = {v:k for k,v in mapping.items()}
return mapping.get(param)
def __get_v8_optimizefor_value(self, quality, reverse=False):
mapping = {
1: 'quality',
2: 'balanced',
3: 'performance'
}
if reverse is True:
mapping = {v:k for k,v in mapping.items()}
return mapping.get(quality, self.DEFAULT_OPTIMIZEFOR)
def __get_v8_routingmode_value(self, mode_type, reverse=False):
mapping = {
'fastest': 'fast',
'shortest': 'short'
}
if reverse is True:
mapping = {v:k for k,v in mapping.items()}
return mapping.get(mode_type, self.DEFAULT_ROUTINGMODE)
def __parse_request_parameters(self, source, mode, data_range, range_type,
options):
quality = self.QUALITY_PARAM_V7
if quality in options.keys():
options[quality] = self.__get_v8_optimizefor_value(options[quality])
filtered_options = {self.__get_v8_param(k): v for k, v in options.iteritems()
if k.lower() in self.OPTIONAL_PARAMS}
filtered_options.update(source)
filtered_options.update(mode)
filtered_options.update({'range[values]': ",".join(map(str, data_range))})
filtered_options.update({'range[type]': range_type})
filtered_options.update({'apikey': self._apikey})
return filtered_options
def __parse_isolines_response(self, response):
parsed_response = json.loads(response)
isolines_response = parsed_response['isolines']
isolines = []
for isoline in isolines_response:
if not isoline['polygons']:
geom_value = []
else:
geom_value = fp.decode(isoline['polygons'][0]['outer'])
isolines.append({'range': isoline['range']['value'],
'geom': geom_value})
return isolines
def __parse_source_param(self, source, options):
key = 'origin'
if 'is_destination' in options and options['is_destination'].lower() == 'true':
key = 'destination'
return {key: source}
def __get_mode_params(self, mode, options):
mode_params = {}
if mode in self.ACCEPTED_MODES:
mode_source = self.ACCEPTED_MODES[mode]
mode_params.update({'transportmode': mode_source})
else:
raise WrongParams("{0} is not an accepted mode type".format(mode))
if 'mode_type' in options:
mode_type = self.__get_v8_routingmode_value(options['mode_type'])
else:
mode_type = self.DEFAULT_ROUTINGMODE
# Do not set routingmode if transportmode value is equivalent to 'walk' mode
if mode_source != self.HERE_WALK_MODE:
mode_params.update({'routingmode': mode_type})
if not ('mode_traffic' in options and options['mode_traffic'] == 'enabled'):
mode_params.update({'departuretime': 'any'})
return mode_params

35
server/lib/python/cartodb_services/cartodb_services/here/service_factory.py Normal file
View File

@ -0,0 +1,35 @@
from cartodb_services.here.geocoder import HereMapsGeocoder, HereMapsGeocoderV7
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder, HereMapsBulkGeocoderV7
from cartodb_services.here.routing import HereMapsRoutingIsoline, HereMapsRoutingIsolineV8
GEOCODING_DEFAULT_MAXRESULTS = 1
def get_geocoder(logger, app_id=None, app_code=None, service_params=None, maxresults=GEOCODING_DEFAULT_MAXRESULTS, use_apikey=False, apikey=None):
if use_apikey is True:
return HereMapsGeocoderV7(apikey=apikey, logger=logger,
service_params=service_params,
limit=maxresults)
else:
return HereMapsGeocoder(app_id=app_id, app_code=app_code,
logger=logger,
service_params=service_params,
maxresults=maxresults)
def get_bulk_geocoder(logger, app_id=None, app_code=None, service_params=None, use_apikey=False, apikey=None):
if use_apikey is True:
return HereMapsBulkGeocoderV7(apikey=apikey, logger=logger,
service_params=service_params)
else:
return HereMapsBulkGeocoder(app_id=app_id, app_code=app_code,
logger=logger,
service_params=service_params)
def get_routing_isoline(logger, app_id=None, app_code=None, service_params=None, use_apikey=False, apikey=None):
if use_apikey is True:
return HereMapsRoutingIsolineV8(apikey=apikey, logger=logger,
service_params=service_params)
else:
return HereMapsRoutingIsoline(app_id=app_id, app_code=app_code,
logger=logger,
service_params=service_params)

5
server/lib/python/cartodb_services/cartodb_services/here/types.py
View File

@ -11,7 +11,10 @@ def geo_polyline_to_multipolygon(polyline):
else:
coordinates = []
for point in polyline:
lat, lon = point.split(',')
if isinstance(point, tuple):
lat, lon = point
else:
lat, lon = point.split(',')
coordinates.append("%s %s" % (lon, lat))
wkt_coordinates = ','.join(coordinates)

48
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -198,6 +198,8 @@ class IsolinesRoutingConfig(ServiceConfig):
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
self._heremaps_apikey = db_config.heremaps_isolines_apikey
self._heremaps_use_apikey = db_config.heremaps_isolines_use_apikey
self._heremaps_service_params = db_config.heremaps_isolines_service_params
elif self._isolines_provider == self.MAPZEN_PROVIDER:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
@ -250,6 +252,14 @@ class IsolinesRoutingConfig(ServiceConfig):
def heremaps_service_params(self):
return self._heremaps_service_params
@property
def heremaps_apikey(self):
return self._heremaps_apikey
@property
def heremaps_use_apikey(self):
return self._heremaps_use_apikey
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@ -373,8 +383,12 @@ class GeocoderConfig(ServiceConfig):
def __check_config(self, filtered_config):
if self._geocoder_provider == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
not self.heremaps_app_id or not self.heremaps_app_code:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Heremaps config is not set up""")
if self.heremaps_use_apikey is True:
if not self.heremaps_apikey:
raise ConfigException("""Heremaps apikey is not set up""")
elif not self.heremaps_app_id or not self.heremaps_app_code:
raise ConfigException("""Heremaps app id or app code is not set up""")
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
@ -412,6 +426,8 @@ class GeocoderConfig(ServiceConfig):
if self._geocoder_provider == self.NOKIA_GEOCODER:
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._heremaps_apikey = db_config.heremaps_geocoder_apikey
self._heremaps_use_apikey = db_config.heremaps_geocoder_use_apikey
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
self._heremaps_service_params = db_config.heremaps_geocoder_service_params
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
@ -505,6 +521,14 @@ class GeocoderConfig(ServiceConfig):
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def heremaps_apikey(self):
return self._heremaps_apikey
@property
def heremaps_use_apikey(self):
return self._heremaps_use_apikey
@property
def heremaps_service_params(self):
return self._heremaps_service_params
@ -593,11 +617,15 @@ class ServicesDBConfig:
heremaps_conf = json.loads(heremaps_conf_json)
self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
self._heremaps_geocoder_apikey = heremaps_conf['geocoder'].get('apikey')
self._heremaps_geocoder_use_apikey = heremaps_conf['geocoder'].get('use_apikey', False)
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
'geocoder_cost_per_hit']
self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
self._heremaps_isolines_apikey = heremaps_conf['isolines'].get('apikey')
self._heremaps_isolines_use_apikey = heremaps_conf['isolines'].get('use_apikey', False)
self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
def _get_mapzen_config(self):
@ -684,6 +712,14 @@ class ServicesDBConfig:
def heremaps_isolines_app_code(self):
return self._heremaps_isolines_app_code
@property
def heremaps_isolines_apikey(self):
return self._heremaps_isolines_apikey
@property
def heremaps_isolines_use_apikey(self):
return self._heremaps_isolines_use_apikey
@property
def heremaps_isolines_service_params(self):
return self._heremaps_isolines_service_params
@ -696,6 +732,14 @@ class ServicesDBConfig:
def heremaps_geocoder_app_code(self):
return self._heremaps_geocoder_app_code
@property
def heremaps_geocoder_apikey(self):
return self._heremaps_geocoder_apikey
@property
def heremaps_geocoder_use_apikey(self):
return self._heremaps_geocoder_use_apikey
@property
def heremaps_geocoder_cost_per_hit(self):
return self._heremaps_geocoder_cost_per_hit

3
server/lib/python/cartodb_services/requirements.txt
View File

@ -11,10 +11,11 @@ rratelimit==0.0.4
botocore==1.18.18
mapbox==0.14.0
pygeocodio==0.11.1
flexpolyline==0.1.0
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7
requests-mock==0.7.0
freezegun==0.3.7
freezegun==0.3.7

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.24.0',
version='0.24.1',
description='CartoDB Services API Python Library',