Merge pull request #622 from CartoDB/feature/ch115970/support-new-here-api-keys
Feature/ch115970/support new here api keys
This commit is contained in:
commit
7862e70bab
@ -132,6 +132,15 @@ SELECT CDB_Conf_SetConf(
|
|||||||
);
|
);
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Use HERE apikeys
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT CDB_Conf_SetConf(
|
||||||
|
'heremaps_conf',
|
||||||
|
'{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true, "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true}}'
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
#### Mapzen configuration
|
#### Mapzen configuration
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
# Makefile to generate the extension out of separate sql source files.
|
# Makefile to generate the extension out of separate sql source files.
|
||||||
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
|
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
|
||||||
EXTENSION = cdb_dataservices_server
|
EXTENSION = cdb_dataservices_server
|
||||||
EXTVERSION = 0.40.0
|
EXTVERSION = 0.40.1
|
||||||
|
|
||||||
# The new version to be generated from templates
|
# The new version to be generated from templates
|
||||||
SED ?= sed
|
SED ?= sed
|
||||||
|
110
server/extension/cdb_dataservices_server--0.40.0--0.40.1.sql
Normal file
110
server/extension/cdb_dataservices_server--0.40.0--0.40.1.sql
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||||
|
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||||
|
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.40.1'" to load this file. \quit
|
||||||
|
|
||||||
|
-- HERE goes your code to upgrade/downgrade
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||||
|
RETURNS Geometry AS $$
|
||||||
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
|
from cartodb_services.tools import QuotaExceededException
|
||||||
|
from cartodb_services.here import get_geocoder
|
||||||
|
|
||||||
|
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||||
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
|
|
||||||
|
try:
|
||||||
|
service_manager.assert_within_limits()
|
||||||
|
geocoder = get_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
|
||||||
|
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||||
|
if coordinates:
|
||||||
|
service_manager.quota_service.increment_success_service_use()
|
||||||
|
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||||
|
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||||
|
return point['st_setsrid']
|
||||||
|
else:
|
||||||
|
service_manager.quota_service.increment_empty_service_use()
|
||||||
|
return None
|
||||||
|
except QuotaExceededException as qe:
|
||||||
|
service_manager.quota_service.increment_failed_service_use()
|
||||||
|
return None
|
||||||
|
except BaseException as e:
|
||||||
|
import sys
|
||||||
|
service_manager.quota_service.increment_failed_service_use()
|
||||||
|
service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||||
|
raise Exception('Error trying to geocode street point using here maps')
|
||||||
|
finally:
|
||||||
|
service_manager.quota_service.increment_total_service_use()
|
||||||
|
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||||
|
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||||
|
from cartodb_services import run_street_point_geocoder
|
||||||
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
|
from cartodb_services.here import get_bulk_geocoder
|
||||||
|
|
||||||
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
|
geocoder = get_bulk_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
|
||||||
|
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||||
|
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
|
||||||
|
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||||
|
import json
|
||||||
|
from cartodb_services.here import get_routing_isoline
|
||||||
|
from cartodb_services.metrics import QuotaService
|
||||||
|
from cartodb_services.here.types import geo_polyline_to_multipolygon
|
||||||
|
from cartodb_services.tools import Logger,LoggerConfig
|
||||||
|
|
||||||
|
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||||
|
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||||
|
|
||||||
|
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||||
|
logger_config = GD["logger_config"]
|
||||||
|
logger = Logger(logger_config)
|
||||||
|
# -- Check the quota
|
||||||
|
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||||
|
if not quota_service.check_user_quota():
|
||||||
|
raise Exception('You have reached the limit of your quota')
|
||||||
|
|
||||||
|
try:
|
||||||
|
client = get_routing_isoline(app_id=user_isolines_routing_config.heremaps_app_id or None,
|
||||||
|
app_code=user_isolines_routing_config.heremaps_app_code or None, logger=logger,
|
||||||
|
apikey=user_isolines_routing_config.heremaps_apikey or None,
|
||||||
|
use_apikey=user_isolines_routing_config.heremaps_use_apikey or False)
|
||||||
|
client_api_version = client.get_api_version()
|
||||||
|
if source:
|
||||||
|
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||||
|
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||||
|
if client_api_version == 7:
|
||||||
|
source_str = 'geo!%f,%f' % (lat, lon)
|
||||||
|
else:
|
||||||
|
source_str = '%f,%f' % (lat, lon)
|
||||||
|
else:
|
||||||
|
source_str = None
|
||||||
|
|
||||||
|
if type == 'isodistance':
|
||||||
|
resp = client.calculate_isodistance(source_str, mode, data_range, options)
|
||||||
|
elif type == 'isochrone':
|
||||||
|
resp = client.calculate_isochrone(source_str, mode, data_range, options)
|
||||||
|
|
||||||
|
if resp:
|
||||||
|
result = []
|
||||||
|
for isoline in resp:
|
||||||
|
data_range_n = isoline['range']
|
||||||
|
polyline = isoline['geom']
|
||||||
|
multipolygon = geo_polyline_to_multipolygon(polyline)
|
||||||
|
result.append([source, data_range_n, multipolygon])
|
||||||
|
quota_service.increment_success_service_use()
|
||||||
|
quota_service.increment_isolines_service_use(len(resp))
|
||||||
|
return result
|
||||||
|
else:
|
||||||
|
quota_service.increment_empty_service_use()
|
||||||
|
return []
|
||||||
|
except BaseException as e:
|
||||||
|
import sys
|
||||||
|
quota_service.increment_failed_service_use()
|
||||||
|
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||||
|
raise Exception('Error trying to get mapzen isolines')
|
||||||
|
finally:
|
||||||
|
quota_service.increment_total_service_use()
|
||||||
|
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
105
server/extension/cdb_dataservices_server--0.40.1--0.40.0.sql
Normal file
105
server/extension/cdb_dataservices_server--0.40.1--0.40.0.sql
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||||
|
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||||
|
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.40.0'" to load this file. \quit
|
||||||
|
|
||||||
|
-- HERE goes your code to upgrade/downgrade
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||||
|
RETURNS Geometry AS $$
|
||||||
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
|
from cartodb_services.tools import QuotaExceededException
|
||||||
|
from cartodb_services.here import HereMapsGeocoder
|
||||||
|
|
||||||
|
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||||
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
|
|
||||||
|
try:
|
||||||
|
service_manager.assert_within_limits()
|
||||||
|
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
|
||||||
|
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||||
|
if coordinates:
|
||||||
|
service_manager.quota_service.increment_success_service_use()
|
||||||
|
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||||
|
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||||
|
return point['st_setsrid']
|
||||||
|
else:
|
||||||
|
service_manager.quota_service.increment_empty_service_use()
|
||||||
|
return None
|
||||||
|
except QuotaExceededException as qe:
|
||||||
|
service_manager.quota_service.increment_failed_service_use()
|
||||||
|
return None
|
||||||
|
except BaseException as e:
|
||||||
|
import sys
|
||||||
|
service_manager.quota_service.increment_failed_service_use()
|
||||||
|
service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||||
|
raise Exception('Error trying to geocode street point using here maps')
|
||||||
|
finally:
|
||||||
|
service_manager.quota_service.increment_total_service_use()
|
||||||
|
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||||
|
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||||
|
from cartodb_services import run_street_point_geocoder
|
||||||
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
|
from cartodb_services.here import HereMapsBulkGeocoder
|
||||||
|
|
||||||
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
|
geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
|
||||||
|
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||||
|
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
|
||||||
|
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||||
|
import json
|
||||||
|
from cartodb_services.here import HereMapsRoutingIsoline
|
||||||
|
from cartodb_services.metrics import QuotaService
|
||||||
|
from cartodb_services.here.types import geo_polyline_to_multipolygon
|
||||||
|
from cartodb_services.tools import Logger,LoggerConfig
|
||||||
|
|
||||||
|
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||||
|
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||||
|
|
||||||
|
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||||
|
logger_config = GD["logger_config"]
|
||||||
|
logger = Logger(logger_config)
|
||||||
|
# -- Check the quota
|
||||||
|
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||||
|
if not quota_service.check_user_quota():
|
||||||
|
raise Exception('You have reached the limit of your quota')
|
||||||
|
|
||||||
|
try:
|
||||||
|
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
|
||||||
|
user_isolines_routing_config.heremaps_app_code, logger)
|
||||||
|
|
||||||
|
if source:
|
||||||
|
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||||
|
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||||
|
source_str = 'geo!%f,%f' % (lat, lon)
|
||||||
|
else:
|
||||||
|
source_str = None
|
||||||
|
|
||||||
|
if type == 'isodistance':
|
||||||
|
resp = client.calculate_isodistance(source_str, mode, data_range, options)
|
||||||
|
elif type == 'isochrone':
|
||||||
|
resp = client.calculate_isochrone(source_str, mode, data_range, options)
|
||||||
|
|
||||||
|
if resp:
|
||||||
|
result = []
|
||||||
|
for isoline in resp:
|
||||||
|
data_range_n = isoline['range']
|
||||||
|
polyline = isoline['geom']
|
||||||
|
multipolygon = geo_polyline_to_multipolygon(polyline)
|
||||||
|
result.append([source, data_range_n, multipolygon])
|
||||||
|
quota_service.increment_success_service_use()
|
||||||
|
quota_service.increment_isolines_service_use(len(resp))
|
||||||
|
return result
|
||||||
|
else:
|
||||||
|
quota_service.increment_empty_service_use()
|
||||||
|
return []
|
||||||
|
except BaseException as e:
|
||||||
|
import sys
|
||||||
|
quota_service.increment_failed_service_use()
|
||||||
|
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||||
|
raise Exception('Error trying to get mapzen isolines')
|
||||||
|
finally:
|
||||||
|
quota_service.increment_total_service_use()
|
||||||
|
$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
2419
server/extension/cdb_dataservices_server--0.40.1.sql
Normal file
2419
server/extension/cdb_dataservices_server--0.40.1.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -125,14 +125,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
|
|||||||
RETURNS Geometry AS $$
|
RETURNS Geometry AS $$
|
||||||
from cartodb_services.tools import LegacyServiceManager
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
from cartodb_services.tools import QuotaExceededException
|
from cartodb_services.tools import QuotaExceededException
|
||||||
from cartodb_services.here import HereMapsGeocoder
|
from cartodb_services.here import get_geocoder
|
||||||
|
|
||||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||||
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
service_manager.assert_within_limits()
|
service_manager.assert_within_limits()
|
||||||
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
|
geocoder = get_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
|
||||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||||
if coordinates:
|
if coordinates:
|
||||||
service_manager.quota_service.increment_success_service_use()
|
service_manager.quota_service.increment_success_service_use()
|
||||||
|
@ -57,10 +57,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_st
|
|||||||
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||||
from cartodb_services import run_street_point_geocoder
|
from cartodb_services import run_street_point_geocoder
|
||||||
from cartodb_services.tools import LegacyServiceManager
|
from cartodb_services.tools import LegacyServiceManager
|
||||||
from cartodb_services.here import HereMapsBulkGeocoder
|
from cartodb_services.here import get_bulk_geocoder
|
||||||
|
|
||||||
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
|
||||||
geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
|
geocoder = get_bulk_geocoder(app_id=service_manager.config.heremaps_app_id or None, app_code=service_manager.config.heremaps_app_code or None, logger=service_manager.logger, service_params=service_manager.config.heremaps_service_params, apikey=service_manager.config.heremaps_apikey or None, use_apikey=service_manager.config.heremaps_use_apikey or False)
|
||||||
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||||
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||||
|
|
||||||
|
@ -3,7 +3,7 @@ CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326),
|
|||||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
|
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
|
||||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||||
import json
|
import json
|
||||||
from cartodb_services.here import HereMapsRoutingIsoline
|
from cartodb_services.here import get_routing_isoline
|
||||||
from cartodb_services.metrics import QuotaService
|
from cartodb_services.metrics import QuotaService
|
||||||
from cartodb_services.here.types import geo_polyline_to_multipolygon
|
from cartodb_services.here.types import geo_polyline_to_multipolygon
|
||||||
from cartodb_services.tools import Logger,LoggerConfig
|
from cartodb_services.tools import Logger,LoggerConfig
|
||||||
@ -20,13 +20,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
|||||||
raise Exception('You have reached the limit of your quota')
|
raise Exception('You have reached the limit of your quota')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
|
client = get_routing_isoline(app_id=user_isolines_routing_config.heremaps_app_id or None,
|
||||||
user_isolines_routing_config.heremaps_app_code, logger)
|
app_code=user_isolines_routing_config.heremaps_app_code or None, logger=logger,
|
||||||
|
apikey=user_isolines_routing_config.heremaps_apikey or None,
|
||||||
|
use_apikey=user_isolines_routing_config.heremaps_use_apikey or False)
|
||||||
|
client_api_version = client.get_api_version()
|
||||||
if source:
|
if source:
|
||||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||||
source_str = 'geo!%f,%f' % (lat, lon)
|
if client_api_version == 7:
|
||||||
|
source_str = 'geo!%f,%f' % (lat, lon)
|
||||||
|
else:
|
||||||
|
source_str = '%f,%f' % (lat, lon)
|
||||||
else:
|
else:
|
||||||
source_str = None
|
source_str = None
|
||||||
|
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
from cartodb_services.here.geocoder import HereMapsGeocoder
|
from cartodb_services.here.geocoder import HereMapsGeocoder, HereMapsGeocoderV7
|
||||||
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
|
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder, HereMapsBulkGeocoderV7
|
||||||
|
from cartodb_services.here.service_factory import get_geocoder, get_bulk_geocoder, get_routing_isoline
|
||||||
from cartodb_services.here.routing import HereMapsRoutingIsoline
|
from cartodb_services.here.routing import HereMapsRoutingIsoline
|
||||||
|
@ -7,7 +7,7 @@ import xml.etree.ElementTree as ET
|
|||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
from cartodb_services import StreetPointBulkGeocoder
|
from cartodb_services import StreetPointBulkGeocoder
|
||||||
from cartodb_services.here import HereMapsGeocoder
|
from cartodb_services.here import HereMapsGeocoder, HereMapsGeocoderV7
|
||||||
from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED
|
from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED
|
||||||
from cartodb_services.metrics import Traceable
|
from cartodb_services.metrics import Traceable
|
||||||
from cartodb_services.tools.exceptions import ServiceException
|
from cartodb_services.tools.exceptions import ServiceException
|
||||||
@ -26,7 +26,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
|
|||||||
JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
|
JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
|
||||||
|
|
||||||
def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
|
def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
|
||||||
HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults)
|
HereMapsGeocoder.__init__(self, app_id=app_id, app_code=app_code, logger=logger, service_params=service_params, maxresults=maxresults)
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
self.session.mount(self.BATCH_URL,
|
self.session.mount(self.BATCH_URL,
|
||||||
HTTPAdapter(max_retries=self.max_retries))
|
HTTPAdapter(max_retries=self.max_retries))
|
||||||
@ -153,3 +153,140 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
|
|||||||
results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
|
results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
class HereMapsBulkGeocoderV7(HereMapsGeocoderV7, StreetPointBulkGeocoder):
|
||||||
|
MAX_BATCH_SIZE = 1000000 # From the docs
|
||||||
|
MIN_BATCHED_SEARCH = 100 # Under this, serial will be used
|
||||||
|
BATCH_URL = 'https://batch.geocoder.ls.hereapi.com/6.2/jobs'
|
||||||
|
# https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
|
||||||
|
META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
|
||||||
|
MAX_STALLED_RETRIES = 100
|
||||||
|
BATCH_RETRY_SLEEP_S = 5
|
||||||
|
JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
|
||||||
|
|
||||||
|
def __init__(self, apikey, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
|
||||||
|
HereMapsGeocoderV7.__init__(self, apikey=apikey, logger=logger, service_params=service_params, limit=maxresults)
|
||||||
|
self.apikey = apikey
|
||||||
|
self.session = requests.Session()
|
||||||
|
self.session.mount(self.BATCH_URL,
|
||||||
|
HTTPAdapter(max_retries=self.max_retries))
|
||||||
|
self.credentials_params = {
|
||||||
|
'apikey': self.apikey
|
||||||
|
}
|
||||||
|
|
||||||
|
def _should_use_batch(self, searches):
|
||||||
|
return len(searches) >= self.MIN_BATCHED_SEARCH
|
||||||
|
|
||||||
|
def _serial_geocode(self, searches):
|
||||||
|
results = []
|
||||||
|
for search in searches:
|
||||||
|
(search_id, address, city, state, country) = search
|
||||||
|
try:
|
||||||
|
result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
|
||||||
|
except Exception as e:
|
||||||
|
self._logger.error("Error geocoding", e)
|
||||||
|
result = geocoder_error_response("Error geocoding")
|
||||||
|
results.append((search_id, result[0], result[1]))
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _batch_geocode(self, searches):
|
||||||
|
request_id = self._send_batch(self._searches_to_csv(searches))
|
||||||
|
|
||||||
|
last_processed = 0
|
||||||
|
stalled_retries = 0
|
||||||
|
# https://developer.here.com/documentation/batch-geocoder/topics/job-status.html
|
||||||
|
while True:
|
||||||
|
job_info = self._job_status(request_id)
|
||||||
|
if job_info.processed_count == last_processed:
|
||||||
|
stalled_retries += 1
|
||||||
|
if stalled_retries > self.MAX_STALLED_RETRIES:
|
||||||
|
raise Exception('Too many retries for job {}'.format(request_id))
|
||||||
|
else:
|
||||||
|
stalled_retries = 0
|
||||||
|
last_processed = job_info.processed_count
|
||||||
|
|
||||||
|
if job_info.status in self.JOB_FINAL_STATES:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
time.sleep(self.BATCH_RETRY_SLEEP_S)
|
||||||
|
|
||||||
|
results = self._download_results(request_id)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _searches_to_csv(self, searches):
|
||||||
|
queue = cStringIO.StringIO()
|
||||||
|
writer = csv.writer(queue, delimiter='|')
|
||||||
|
writer.writerow(['recId', 'searchText', 'country'])
|
||||||
|
|
||||||
|
for search in searches:
|
||||||
|
fields = [search.address, search.city, search.state]
|
||||||
|
search_text = ', '.join(filter(None, fields))
|
||||||
|
row = [s.encode("utf-8") if s else ''
|
||||||
|
for s in [str(search.id), search_text, search.country]]
|
||||||
|
writer.writerow(row)
|
||||||
|
|
||||||
|
return queue.getvalue()
|
||||||
|
|
||||||
|
def _send_batch(self, data):
|
||||||
|
cols = 'displayLatitude,displayLongitude,' + ','.join(self.META_COLS)
|
||||||
|
request_params = self.credentials_params.copy()
|
||||||
|
request_params.update({
|
||||||
|
'gen': 8,
|
||||||
|
'action': 'run',
|
||||||
|
'header': 'true',
|
||||||
|
'inDelim': '|',
|
||||||
|
'outDelim': '|',
|
||||||
|
'outCols': cols,
|
||||||
|
'outputcombined': 'true'
|
||||||
|
})
|
||||||
|
|
||||||
|
response = self.session.post(self.BATCH_URL, data=data,
|
||||||
|
params=request_params,
|
||||||
|
timeout=(self.connect_timeout, self.read_timeout))
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
root = ET.fromstring(response.text)
|
||||||
|
return root.find('./Response/MetaInfo/RequestId').text
|
||||||
|
else:
|
||||||
|
raise ServiceException("Error sending HERE batch", response)
|
||||||
|
|
||||||
|
def _job_status(self, request_id):
|
||||||
|
polling_params = self.credentials_params.copy()
|
||||||
|
polling_params.update({'action': 'status'})
|
||||||
|
polling_r = self.session.get("{}/{}".format(self.BATCH_URL, request_id),
|
||||||
|
params=polling_params,
|
||||||
|
timeout=(self.connect_timeout, self.read_timeout))
|
||||||
|
polling_root = ET.fromstring(polling_r.text)
|
||||||
|
return HereJobStatus(
|
||||||
|
total_count=int(polling_root.find('./Response/TotalCount').text),
|
||||||
|
processed_count=int(polling_root.find('./Response/ProcessedCount').text),
|
||||||
|
status=polling_root.find('./Response/Status').text)
|
||||||
|
|
||||||
|
def _download_results(self, job_id):
|
||||||
|
result_r = self.session.get("{}/{}/all".format(self.BATCH_URL, job_id),
|
||||||
|
params=self.credentials_params,
|
||||||
|
timeout=(self.connect_timeout, self.read_timeout))
|
||||||
|
root_zip = zipfile.ZipFile(io.BytesIO(result_r.content))
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for name in root_zip.namelist():
|
||||||
|
if name.endswith('_out.txt'):
|
||||||
|
reader = csv.DictReader(root_zip.open(name), delimiter='|')
|
||||||
|
for row in reader:
|
||||||
|
if row['SeqNumber'] == '1': # First per requested data
|
||||||
|
precision = self.PRECISION_BY_MATCH_TYPE.get(
|
||||||
|
row.get('matchType'), PRECISION_INTERPOLATED)
|
||||||
|
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None)
|
||||||
|
results.append((row['recId'],
|
||||||
|
[row['displayLongitude'], row['displayLatitude']],
|
||||||
|
geocoder_metadata(
|
||||||
|
float(row['relevance']),
|
||||||
|
precision,
|
||||||
|
[match_type] if match_type else []
|
||||||
|
)))
|
||||||
|
elif row['matchLevel'] == 'NOMATCH':
|
||||||
|
results.append((row['recId'], [], {}))
|
||||||
|
elif row['matchLevel'] == 'FAILED':
|
||||||
|
results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
|
||||||
|
return results
|
||||||
|
@ -8,9 +8,10 @@ from requests.adapters import HTTPAdapter
|
|||||||
from exceptions import *
|
from exceptions import *
|
||||||
from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE
|
from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE
|
||||||
from cartodb_services.metrics import Traceable
|
from cartodb_services.metrics import Traceable
|
||||||
|
from cartodb_services.tools.country import country_to_iso3
|
||||||
|
|
||||||
class HereMapsGeocoder(Traceable):
|
class HereMapsGeocoder(Traceable):
|
||||||
'A Here Maps Geocoder wrapper for python'
|
'A Here Maps v6 Geocoder wrapper for python'
|
||||||
|
|
||||||
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
|
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
|
||||||
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
|
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
|
||||||
@ -30,7 +31,7 @@ class HereMapsGeocoder(Traceable):
|
|||||||
'searchtext',
|
'searchtext',
|
||||||
'state',
|
'state',
|
||||||
'street'
|
'street'
|
||||||
]
|
]
|
||||||
|
|
||||||
ADMITTED_PARAMS = [
|
ADMITTED_PARAMS = [
|
||||||
'additionaldata',
|
'additionaldata',
|
||||||
@ -50,7 +51,7 @@ class HereMapsGeocoder(Traceable):
|
|||||||
'politicalview',
|
'politicalview',
|
||||||
'prox',
|
'prox',
|
||||||
'strictlanguagemode'
|
'strictlanguagemode'
|
||||||
] + ADDRESS_PARAMS
|
] + ADDRESS_PARAMS
|
||||||
|
|
||||||
PRECISION_BY_MATCH_TYPE = {
|
PRECISION_BY_MATCH_TYPE = {
|
||||||
'pointAddress': PRECISION_PRECISE,
|
'pointAddress': PRECISION_PRECISE,
|
||||||
@ -151,3 +152,181 @@ class HereMapsGeocoder(Traceable):
|
|||||||
precision,
|
precision,
|
||||||
[match_type] if match_type else []
|
[match_type] if match_type else []
|
||||||
)
|
)
|
||||||
|
|
||||||
|
class HereMapsGeocoderV7(Traceable):
|
||||||
|
'A Here Maps Geocoder v7 wrapper for python'
|
||||||
|
|
||||||
|
PRODUCTION_GEOCODE_JSON_URL = 'https://geocode.search.hereapi.com/v1/geocode'
|
||||||
|
DEFAULT_MAXRESULTS = 1
|
||||||
|
READ_TIMEOUT = 60
|
||||||
|
CONNECT_TIMEOUT = 10
|
||||||
|
MAX_RETRIES=1
|
||||||
|
COUNTRY_ISO3_LENGTH = 3
|
||||||
|
|
||||||
|
ADDRESS_PARAMS = [
|
||||||
|
'city',
|
||||||
|
'country',
|
||||||
|
'county',
|
||||||
|
'district',
|
||||||
|
'housenumber',
|
||||||
|
'postalcode',
|
||||||
|
'searchtext',
|
||||||
|
'state',
|
||||||
|
'street'
|
||||||
|
]
|
||||||
|
|
||||||
|
QUALIFIED_PARAMS = [
|
||||||
|
'state',
|
||||||
|
'city',
|
||||||
|
'county',
|
||||||
|
'district',
|
||||||
|
'postalcode',
|
||||||
|
'housenumber',
|
||||||
|
'street'
|
||||||
|
]
|
||||||
|
|
||||||
|
PRECISION_BY_MATCH_TYPE = {
|
||||||
|
'PA': PRECISION_PRECISE,
|
||||||
|
'interpolated': PRECISION_INTERPOLATED
|
||||||
|
}
|
||||||
|
MATCH_TYPE_BY_MATCH_LEVEL = {
|
||||||
|
'landmark': 'point_of_interest',
|
||||||
|
'country': 'country',
|
||||||
|
'state': 'state',
|
||||||
|
'county': 'county',
|
||||||
|
'city': 'locality',
|
||||||
|
'district': 'district',
|
||||||
|
'street': 'street',
|
||||||
|
'intersection': 'intersection',
|
||||||
|
'houseNumber': 'street_number',
|
||||||
|
'postalCode': 'postal_code'
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, apikey, logger, service_params=None, limit=DEFAULT_MAXRESULTS):
|
||||||
|
service_params = service_params or {}
|
||||||
|
self.apikey = apikey
|
||||||
|
self._logger = logger
|
||||||
|
self.limit = limit
|
||||||
|
self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL)
|
||||||
|
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||||
|
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||||
|
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||||
|
|
||||||
|
def geocode(self, **kwargs):
|
||||||
|
return self.geocode_meta(**kwargs)[0]
|
||||||
|
|
||||||
|
def geocode_meta(self, **kwargs):
|
||||||
|
params = {}
|
||||||
|
for key, value in kwargs.iteritems():
|
||||||
|
if value and value.strip():
|
||||||
|
params[key] = value
|
||||||
|
if not params:
|
||||||
|
return EMPTY_RESPONSE
|
||||||
|
return self._execute_geocode(params)
|
||||||
|
|
||||||
|
def _execute_geocode(self, params):
|
||||||
|
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
|
||||||
|
raise BadGeocodingParams(params)
|
||||||
|
try:
|
||||||
|
response = self._perform_request(params)
|
||||||
|
result = response['items'][0]
|
||||||
|
return [self._extract_lng_lat_from_result(result),
|
||||||
|
self._extract_metadata_from_result(result)]
|
||||||
|
except IndexError:
|
||||||
|
return EMPTY_RESPONSE
|
||||||
|
except KeyError as e:
|
||||||
|
self._logger.error('params: {}'.format(params), e)
|
||||||
|
raise MalformedResult()
|
||||||
|
|
||||||
|
def _get_v7param(self, param, reverse=False):
|
||||||
|
mapping = {
|
||||||
|
'city': 'city',
|
||||||
|
'country': 'in',
|
||||||
|
'county': 'county',
|
||||||
|
'district': 'district',
|
||||||
|
'housenumber': 'houseNumber',
|
||||||
|
'postalcode': 'postalCode',
|
||||||
|
'state': 'state',
|
||||||
|
'street': 'street',
|
||||||
|
'searchtext': 'q'
|
||||||
|
}
|
||||||
|
|
||||||
|
if reverse is True:
|
||||||
|
mapping = {v:k for k,v in mapping.items()}
|
||||||
|
|
||||||
|
return mapping.get(param)
|
||||||
|
|
||||||
|
def _is_iso3_country(self, country):
|
||||||
|
return len(country) == self.COUNTRY_ISO3_LENGTH
|
||||||
|
|
||||||
|
def _parse_country(self, country):
|
||||||
|
country_iso3 = country_to_iso3(country) or country
|
||||||
|
|
||||||
|
parsed_country = "countryCode:{0}".format(country_iso3) if (country_iso3 and self._is_iso3_country(country_iso3)) else None
|
||||||
|
|
||||||
|
return parsed_country
|
||||||
|
|
||||||
|
def _get_qq(self, params):
|
||||||
|
qualified_params = ['{0}={1}'.format(self._get_v7param(k), v) for k,v in params.items() if k in self.QUALIFIED_PARAMS]
|
||||||
|
|
||||||
|
qq = '&'.join(qualified_params) if qualified_params else None
|
||||||
|
|
||||||
|
return qq
|
||||||
|
|
||||||
|
def _parse_params(self, params):
|
||||||
|
country, q = (params.get(value) for value in ('country', 'searchtext'))
|
||||||
|
|
||||||
|
parsed_params = {'q': q}
|
||||||
|
|
||||||
|
qq = self._get_qq(params)
|
||||||
|
|
||||||
|
parsed_country = self._parse_country(country)
|
||||||
|
|
||||||
|
parsed_params.update({k:v for k, v in (('qq', qq), ('in', parsed_country)) if v is not None})
|
||||||
|
|
||||||
|
return parsed_params
|
||||||
|
|
||||||
|
def _perform_request(self, params):
|
||||||
|
request_params = {
|
||||||
|
'apikey': self.apikey,
|
||||||
|
'limit': self.limit,
|
||||||
|
}
|
||||||
|
parsed_params = self._parse_params(params)
|
||||||
|
request_params.update(parsed_params)
|
||||||
|
# TODO Extract HTTP client wrapper
|
||||||
|
session = requests.Session()
|
||||||
|
session.mount(self.host, HTTPAdapter(max_retries=self.max_retries))
|
||||||
|
response = session.get(self.host, params=request_params,
|
||||||
|
timeout=(self.connect_timeout, self.read_timeout))
|
||||||
|
self.add_response_data(response, self._logger)
|
||||||
|
if response.status_code == requests.codes.ok:
|
||||||
|
return json.loads(response.text)
|
||||||
|
elif response.status_code == requests.codes.bad_request:
|
||||||
|
self._logger.warning('Error 4xx trying to geocode street using HERE',
|
||||||
|
data={"response": response.json(), "params":
|
||||||
|
parsed_params})
|
||||||
|
return EMPTY_RESPONSE
|
||||||
|
else:
|
||||||
|
self._logger.error('Error trying to geocode street using HERE',
|
||||||
|
data={"response": response.json(), "params":
|
||||||
|
parsed_params})
|
||||||
|
raise Exception('Error trying to geocode street using Here')
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_lng_lat_from_result(self, result):
|
||||||
|
position = result['position']
|
||||||
|
longitude = position['lng']
|
||||||
|
latitude = position['lat']
|
||||||
|
|
||||||
|
return [longitude, latitude]
|
||||||
|
|
||||||
|
def _extract_metadata_from_result(self, result):
|
||||||
|
# See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses
|
||||||
|
precision = self.PRECISION_BY_MATCH_TYPE.get(
|
||||||
|
result.get('houseNumberType'), PRECISION_INTERPOLATED)
|
||||||
|
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result.get('resultType'), None)
|
||||||
|
return geocoder_metadata(
|
||||||
|
result['scoring']['queryScore'],
|
||||||
|
precision,
|
||||||
|
[match_type] if match_type else []
|
||||||
|
)
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
import requests
|
import requests
|
||||||
import json
|
import json
|
||||||
|
import flexpolyline as fp
|
||||||
|
|
||||||
from cartodb_services.here.exceptions import WrongParams
|
from cartodb_services.here.exceptions import WrongParams
|
||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
@ -7,11 +8,12 @@ from cartodb_services.metrics import Traceable
|
|||||||
|
|
||||||
|
|
||||||
class HereMapsRoutingIsoline(Traceable):
|
class HereMapsRoutingIsoline(Traceable):
|
||||||
'A Here Maps Routing wrapper for python'
|
'A Here Maps Routing v7 wrapper for python'
|
||||||
|
|
||||||
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
|
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
|
||||||
STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
|
STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
|
||||||
ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
|
ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
|
||||||
|
API_VERSION = 7
|
||||||
READ_TIMEOUT = 60
|
READ_TIMEOUT = 60
|
||||||
CONNECT_TIMEOUT = 10
|
CONNECT_TIMEOUT = 10
|
||||||
MAX_RETRIES = 1
|
MAX_RETRIES = 1
|
||||||
@ -42,6 +44,9 @@ class HereMapsRoutingIsoline(Traceable):
|
|||||||
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||||
self._url = "{0}{1}".format(base_url, isoline_path)
|
self._url = "{0}{1}".format(base_url, isoline_path)
|
||||||
|
|
||||||
|
def get_api_version(self):
|
||||||
|
return self.API_VERSION
|
||||||
|
|
||||||
def calculate_isodistance(self, source, mode, data_range, options=[]):
|
def calculate_isodistance(self, source, mode, data_range, options=[]):
|
||||||
return self.__calculate_isolines(source, mode, data_range, 'distance',
|
return self.__calculate_isolines(source, mode, data_range, 'distance',
|
||||||
options)
|
options)
|
||||||
@ -51,7 +56,8 @@ class HereMapsRoutingIsoline(Traceable):
|
|||||||
options)
|
options)
|
||||||
|
|
||||||
def __calculate_isolines(self, source, mode, data_range, range_type,
|
def __calculate_isolines(self, source, mode, data_range, range_type,
|
||||||
options=[]):
|
options=None):
|
||||||
|
options = options or []
|
||||||
parsed_options = self.__parse_options(options)
|
parsed_options = self.__parse_options(options)
|
||||||
source_param = self.__parse_source_param(source, parsed_options)
|
source_param = self.__parse_source_param(source, parsed_options)
|
||||||
mode_param = self.__parse_mode_param(mode, parsed_options)
|
mode_param = self.__parse_mode_param(mode, parsed_options)
|
||||||
@ -150,3 +156,195 @@ class HereMapsRoutingIsoline(Traceable):
|
|||||||
mode_param = "{0};{1}".format(mode_param, mode_feature)
|
mode_param = "{0};{1}".format(mode_param, mode_feature)
|
||||||
|
|
||||||
return {'mode': mode_param}
|
return {'mode': mode_param}
|
||||||
|
|
||||||
|
class HereMapsRoutingIsolineV8(Traceable):
|
||||||
|
'A Here Maps Routing v8 wrapper for python'
|
||||||
|
|
||||||
|
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.router.hereapi.com/v8/isolines'
|
||||||
|
API_VERSION = 8
|
||||||
|
READ_TIMEOUT = 60
|
||||||
|
CONNECT_TIMEOUT = 10
|
||||||
|
MAX_RETRIES = 1
|
||||||
|
QUALITY_PARAM_V7 = 'quality'
|
||||||
|
DEFAULT_OPTIMIZEFOR = 'quality'
|
||||||
|
DEFAULT_ROUTINGMODE = 'short'
|
||||||
|
HERE_WALK_MODE = "pedestrian"
|
||||||
|
HERE_CAR_MODE = "car"
|
||||||
|
|
||||||
|
ACCEPTED_MODES = {
|
||||||
|
"walk": HERE_WALK_MODE,
|
||||||
|
"car": HERE_CAR_MODE
|
||||||
|
}
|
||||||
|
|
||||||
|
OPTIONAL_PARAMS = [
|
||||||
|
'departure',
|
||||||
|
'arrival',
|
||||||
|
'maxpoints',
|
||||||
|
'quality'
|
||||||
|
]
|
||||||
|
|
||||||
|
def __init__(self, apikey, logger, service_params=None):
|
||||||
|
service_params = service_params or {}
|
||||||
|
self._apikey = apikey
|
||||||
|
self._logger = logger
|
||||||
|
self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
|
||||||
|
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||||
|
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||||
|
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||||
|
|
||||||
|
def get_api_version(self):
|
||||||
|
return self.API_VERSION
|
||||||
|
|
||||||
|
def calculate_isodistance(self, source, mode, data_range, options=None):
|
||||||
|
options = options or []
|
||||||
|
return self.__calculate_isolines(source, mode, data_range, 'distance',
|
||||||
|
options)
|
||||||
|
|
||||||
|
def calculate_isochrone(self, source, mode, data_range, options=None):
|
||||||
|
options = options or []
|
||||||
|
return self.__calculate_isolines(source, mode, data_range, 'time',
|
||||||
|
options)
|
||||||
|
|
||||||
|
def __calculate_isolines(self, source, mode, data_range, range_type,
|
||||||
|
options=None):
|
||||||
|
options = options or []
|
||||||
|
parsed_options = self.__parse_options(options)
|
||||||
|
source_param = self.__parse_source_param(source, parsed_options)
|
||||||
|
mode_params = self.__get_mode_params(mode, parsed_options)
|
||||||
|
request_params = self.__parse_request_parameters(source_param,
|
||||||
|
mode_params,
|
||||||
|
data_range,
|
||||||
|
range_type,
|
||||||
|
parsed_options)
|
||||||
|
# TODO Extract HTTP client wrapper
|
||||||
|
session = requests.Session()
|
||||||
|
session.mount(self._url, HTTPAdapter(max_retries=self.max_retries))
|
||||||
|
response = requests.get(self._url, params=request_params,
|
||||||
|
timeout=(self.connect_timeout, self.read_timeout))
|
||||||
|
self.add_response_data(response, self._logger)
|
||||||
|
if response.status_code == requests.codes.ok:
|
||||||
|
return self.__parse_isolines_response(response.text)
|
||||||
|
elif response.status_code == requests.codes.bad_request:
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
self._logger.error('Error trying to calculate HERE isolines',
|
||||||
|
data={"response_status": response.status_code,
|
||||||
|
"response_reason": response.reason,
|
||||||
|
"response_content": response.text,
|
||||||
|
"reponse_url": response.url,
|
||||||
|
"response_headers": response.headers,
|
||||||
|
"source": source, "mode": mode,
|
||||||
|
"data_range": data_range,
|
||||||
|
"range_type": range_type,
|
||||||
|
"options": options})
|
||||||
|
raise Exception('Error trying to calculate HERE isolines')
|
||||||
|
|
||||||
|
def __parse_options(self, options):
|
||||||
|
return dict(option.split('=') for option in options)
|
||||||
|
|
||||||
|
def __get_v8_param(self, param, reverse=False):
|
||||||
|
mapping = {
|
||||||
|
'range': 'range[values]',
|
||||||
|
'rangetype': 'range[type]',
|
||||||
|
'mode': {
|
||||||
|
'type': 'routingmode',
|
||||||
|
'transportmodes': 'transportmode',
|
||||||
|
'trafficmode': None,
|
||||||
|
'feature': 'avoid[feature]'
|
||||||
|
},
|
||||||
|
'quality': 'optimizefor',
|
||||||
|
'maxpoints': 'shape[maxpoints]',
|
||||||
|
'start': 'origin',
|
||||||
|
'destination': 'destination',
|
||||||
|
'departure': 'departuretime',
|
||||||
|
'arrival': 'arrivaltime'
|
||||||
|
}
|
||||||
|
|
||||||
|
if reverse is True:
|
||||||
|
mapping = {v:k for k,v in mapping.items()}
|
||||||
|
|
||||||
|
return mapping.get(param)
|
||||||
|
|
||||||
|
def __get_v8_optimizefor_value(self, quality, reverse=False):
|
||||||
|
mapping = {
|
||||||
|
1: 'quality',
|
||||||
|
2: 'balanced',
|
||||||
|
3: 'performance'
|
||||||
|
}
|
||||||
|
|
||||||
|
if reverse is True:
|
||||||
|
mapping = {v:k for k,v in mapping.items()}
|
||||||
|
|
||||||
|
return mapping.get(quality, self.DEFAULT_OPTIMIZEFOR)
|
||||||
|
|
||||||
|
def __get_v8_routingmode_value(self, mode_type, reverse=False):
|
||||||
|
mapping = {
|
||||||
|
'fastest': 'fast',
|
||||||
|
'shortest': 'short'
|
||||||
|
}
|
||||||
|
|
||||||
|
if reverse is True:
|
||||||
|
mapping = {v:k for k,v in mapping.items()}
|
||||||
|
|
||||||
|
return mapping.get(mode_type, self.DEFAULT_ROUTINGMODE)
|
||||||
|
|
||||||
|
def __parse_request_parameters(self, source, mode, data_range, range_type,
|
||||||
|
options):
|
||||||
|
quality = self.QUALITY_PARAM_V7
|
||||||
|
|
||||||
|
if quality in options.keys():
|
||||||
|
options[quality] = self.__get_v8_optimizefor_value(options[quality])
|
||||||
|
|
||||||
|
filtered_options = {self.__get_v8_param(k): v for k, v in options.iteritems()
|
||||||
|
if k.lower() in self.OPTIONAL_PARAMS}
|
||||||
|
|
||||||
|
filtered_options.update(source)
|
||||||
|
filtered_options.update(mode)
|
||||||
|
filtered_options.update({'range[values]': ",".join(map(str, data_range))})
|
||||||
|
filtered_options.update({'range[type]': range_type})
|
||||||
|
filtered_options.update({'apikey': self._apikey})
|
||||||
|
|
||||||
|
return filtered_options
|
||||||
|
|
||||||
|
def __parse_isolines_response(self, response):
|
||||||
|
parsed_response = json.loads(response)
|
||||||
|
isolines_response = parsed_response['isolines']
|
||||||
|
isolines = []
|
||||||
|
for isoline in isolines_response:
|
||||||
|
if not isoline['polygons']:
|
||||||
|
geom_value = []
|
||||||
|
else:
|
||||||
|
geom_value = fp.decode(isoline['polygons'][0]['outer'])
|
||||||
|
isolines.append({'range': isoline['range']['value'],
|
||||||
|
'geom': geom_value})
|
||||||
|
|
||||||
|
return isolines
|
||||||
|
|
||||||
|
def __parse_source_param(self, source, options):
|
||||||
|
key = 'origin'
|
||||||
|
if 'is_destination' in options and options['is_destination'].lower() == 'true':
|
||||||
|
key = 'destination'
|
||||||
|
|
||||||
|
return {key: source}
|
||||||
|
|
||||||
|
def __get_mode_params(self, mode, options):
|
||||||
|
mode_params = {}
|
||||||
|
if mode in self.ACCEPTED_MODES:
|
||||||
|
mode_source = self.ACCEPTED_MODES[mode]
|
||||||
|
mode_params.update({'transportmode': mode_source})
|
||||||
|
else:
|
||||||
|
raise WrongParams("{0} is not an accepted mode type".format(mode))
|
||||||
|
|
||||||
|
if 'mode_type' in options:
|
||||||
|
mode_type = self.__get_v8_routingmode_value(options['mode_type'])
|
||||||
|
else:
|
||||||
|
mode_type = self.DEFAULT_ROUTINGMODE
|
||||||
|
|
||||||
|
# Do not set routingmode if transportmode value is equivalent to 'walk' mode
|
||||||
|
if mode_source != self.HERE_WALK_MODE:
|
||||||
|
mode_params.update({'routingmode': mode_type})
|
||||||
|
|
||||||
|
if not ('mode_traffic' in options and options['mode_traffic'] == 'enabled'):
|
||||||
|
mode_params.update({'departuretime': 'any'})
|
||||||
|
|
||||||
|
return mode_params
|
||||||
|
@ -0,0 +1,35 @@
|
|||||||
|
from cartodb_services.here.geocoder import HereMapsGeocoder, HereMapsGeocoderV7
|
||||||
|
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder, HereMapsBulkGeocoderV7
|
||||||
|
from cartodb_services.here.routing import HereMapsRoutingIsoline, HereMapsRoutingIsolineV8
|
||||||
|
|
||||||
|
GEOCODING_DEFAULT_MAXRESULTS = 1
|
||||||
|
|
||||||
|
def get_geocoder(logger, app_id=None, app_code=None, service_params=None, maxresults=GEOCODING_DEFAULT_MAXRESULTS, use_apikey=False, apikey=None):
|
||||||
|
if use_apikey is True:
|
||||||
|
return HereMapsGeocoderV7(apikey=apikey, logger=logger,
|
||||||
|
service_params=service_params,
|
||||||
|
limit=maxresults)
|
||||||
|
else:
|
||||||
|
return HereMapsGeocoder(app_id=app_id, app_code=app_code,
|
||||||
|
logger=logger,
|
||||||
|
service_params=service_params,
|
||||||
|
maxresults=maxresults)
|
||||||
|
|
||||||
|
|
||||||
|
def get_bulk_geocoder(logger, app_id=None, app_code=None, service_params=None, use_apikey=False, apikey=None):
|
||||||
|
if use_apikey is True:
|
||||||
|
return HereMapsBulkGeocoderV7(apikey=apikey, logger=logger,
|
||||||
|
service_params=service_params)
|
||||||
|
else:
|
||||||
|
return HereMapsBulkGeocoder(app_id=app_id, app_code=app_code,
|
||||||
|
logger=logger,
|
||||||
|
service_params=service_params)
|
||||||
|
|
||||||
|
def get_routing_isoline(logger, app_id=None, app_code=None, service_params=None, use_apikey=False, apikey=None):
|
||||||
|
if use_apikey is True:
|
||||||
|
return HereMapsRoutingIsolineV8(apikey=apikey, logger=logger,
|
||||||
|
service_params=service_params)
|
||||||
|
else:
|
||||||
|
return HereMapsRoutingIsoline(app_id=app_id, app_code=app_code,
|
||||||
|
logger=logger,
|
||||||
|
service_params=service_params)
|
@ -11,7 +11,10 @@ def geo_polyline_to_multipolygon(polyline):
|
|||||||
else:
|
else:
|
||||||
coordinates = []
|
coordinates = []
|
||||||
for point in polyline:
|
for point in polyline:
|
||||||
lat, lon = point.split(',')
|
if isinstance(point, tuple):
|
||||||
|
lat, lon = point
|
||||||
|
else:
|
||||||
|
lat, lon = point.split(',')
|
||||||
coordinates.append("%s %s" % (lon, lat))
|
coordinates.append("%s %s" % (lon, lat))
|
||||||
wkt_coordinates = ','.join(coordinates)
|
wkt_coordinates = ','.join(coordinates)
|
||||||
|
|
||||||
|
@ -198,6 +198,8 @@ class IsolinesRoutingConfig(ServiceConfig):
|
|||||||
if self._isolines_provider == self.HEREMAPS_PROVIDER:
|
if self._isolines_provider == self.HEREMAPS_PROVIDER:
|
||||||
self._heremaps_app_id = db_config.heremaps_isolines_app_id
|
self._heremaps_app_id = db_config.heremaps_isolines_app_id
|
||||||
self._heremaps_app_code = db_config.heremaps_isolines_app_code
|
self._heremaps_app_code = db_config.heremaps_isolines_app_code
|
||||||
|
self._heremaps_apikey = db_config.heremaps_isolines_apikey
|
||||||
|
self._heremaps_use_apikey = db_config.heremaps_isolines_use_apikey
|
||||||
self._heremaps_service_params = db_config.heremaps_isolines_service_params
|
self._heremaps_service_params = db_config.heremaps_isolines_service_params
|
||||||
elif self._isolines_provider == self.MAPZEN_PROVIDER:
|
elif self._isolines_provider == self.MAPZEN_PROVIDER:
|
||||||
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
|
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
|
||||||
@ -250,6 +252,14 @@ class IsolinesRoutingConfig(ServiceConfig):
|
|||||||
def heremaps_service_params(self):
|
def heremaps_service_params(self):
|
||||||
return self._heremaps_service_params
|
return self._heremaps_service_params
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_apikey(self):
|
||||||
|
return self._heremaps_apikey
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_use_apikey(self):
|
||||||
|
return self._heremaps_use_apikey
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def mapzen_matrix_api_key(self):
|
def mapzen_matrix_api_key(self):
|
||||||
return self._mapzen_matrix_api_key
|
return self._mapzen_matrix_api_key
|
||||||
@ -373,8 +383,12 @@ class GeocoderConfig(ServiceConfig):
|
|||||||
|
|
||||||
def __check_config(self, filtered_config):
|
def __check_config(self, filtered_config):
|
||||||
if self._geocoder_provider == self.NOKIA_GEOCODER:
|
if self._geocoder_provider == self.NOKIA_GEOCODER:
|
||||||
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
|
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
|
||||||
not self.heremaps_app_id or not self.heremaps_app_code:
|
raise ConfigException("""Heremaps config is not set up""")
|
||||||
|
if self.heremaps_use_apikey is True:
|
||||||
|
if not self.heremaps_apikey:
|
||||||
|
raise ConfigException("""Heremaps apikey is not set up""")
|
||||||
|
elif not self.heremaps_app_id or not self.heremaps_app_code:
|
||||||
raise ConfigException("""Heremaps app id or app code is not set up""")
|
raise ConfigException("""Heremaps app id or app code is not set up""")
|
||||||
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
|
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
|
||||||
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
||||||
@ -412,6 +426,8 @@ class GeocoderConfig(ServiceConfig):
|
|||||||
if self._geocoder_provider == self.NOKIA_GEOCODER:
|
if self._geocoder_provider == self.NOKIA_GEOCODER:
|
||||||
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
|
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
|
||||||
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
|
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
|
||||||
|
self._heremaps_apikey = db_config.heremaps_geocoder_apikey
|
||||||
|
self._heremaps_use_apikey = db_config.heremaps_geocoder_use_apikey
|
||||||
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
|
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
|
||||||
self._heremaps_service_params = db_config.heremaps_geocoder_service_params
|
self._heremaps_service_params = db_config.heremaps_geocoder_service_params
|
||||||
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
|
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
|
||||||
@ -505,6 +521,14 @@ class GeocoderConfig(ServiceConfig):
|
|||||||
def heremaps_app_code(self):
|
def heremaps_app_code(self):
|
||||||
return self._heremaps_app_code
|
return self._heremaps_app_code
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_apikey(self):
|
||||||
|
return self._heremaps_apikey
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_use_apikey(self):
|
||||||
|
return self._heremaps_use_apikey
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def heremaps_service_params(self):
|
def heremaps_service_params(self):
|
||||||
return self._heremaps_service_params
|
return self._heremaps_service_params
|
||||||
@ -593,11 +617,15 @@ class ServicesDBConfig:
|
|||||||
heremaps_conf = json.loads(heremaps_conf_json)
|
heremaps_conf = json.loads(heremaps_conf_json)
|
||||||
self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
|
self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
|
||||||
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
|
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
|
||||||
|
self._heremaps_geocoder_apikey = heremaps_conf['geocoder'].get('apikey')
|
||||||
|
self._heremaps_geocoder_use_apikey = heremaps_conf['geocoder'].get('use_apikey', False)
|
||||||
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
|
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
|
||||||
'geocoder_cost_per_hit']
|
'geocoder_cost_per_hit']
|
||||||
self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
|
self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
|
||||||
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
|
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
|
||||||
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
|
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
|
||||||
|
self._heremaps_isolines_apikey = heremaps_conf['isolines'].get('apikey')
|
||||||
|
self._heremaps_isolines_use_apikey = heremaps_conf['isolines'].get('use_apikey', False)
|
||||||
self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
|
self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
|
||||||
|
|
||||||
def _get_mapzen_config(self):
|
def _get_mapzen_config(self):
|
||||||
@ -684,6 +712,14 @@ class ServicesDBConfig:
|
|||||||
def heremaps_isolines_app_code(self):
|
def heremaps_isolines_app_code(self):
|
||||||
return self._heremaps_isolines_app_code
|
return self._heremaps_isolines_app_code
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_isolines_apikey(self):
|
||||||
|
return self._heremaps_isolines_apikey
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_isolines_use_apikey(self):
|
||||||
|
return self._heremaps_isolines_use_apikey
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def heremaps_isolines_service_params(self):
|
def heremaps_isolines_service_params(self):
|
||||||
return self._heremaps_isolines_service_params
|
return self._heremaps_isolines_service_params
|
||||||
@ -696,6 +732,14 @@ class ServicesDBConfig:
|
|||||||
def heremaps_geocoder_app_code(self):
|
def heremaps_geocoder_app_code(self):
|
||||||
return self._heremaps_geocoder_app_code
|
return self._heremaps_geocoder_app_code
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_geocoder_apikey(self):
|
||||||
|
return self._heremaps_geocoder_apikey
|
||||||
|
|
||||||
|
@property
|
||||||
|
def heremaps_geocoder_use_apikey(self):
|
||||||
|
return self._heremaps_geocoder_use_apikey
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def heremaps_geocoder_cost_per_hit(self):
|
def heremaps_geocoder_cost_per_hit(self):
|
||||||
return self._heremaps_geocoder_cost_per_hit
|
return self._heremaps_geocoder_cost_per_hit
|
||||||
|
@ -11,10 +11,11 @@ rratelimit==0.0.4
|
|||||||
botocore==1.18.18
|
botocore==1.18.18
|
||||||
mapbox==0.14.0
|
mapbox==0.14.0
|
||||||
pygeocodio==0.11.1
|
pygeocodio==0.11.1
|
||||||
|
flexpolyline==0.1.0
|
||||||
|
|
||||||
# Test
|
# Test
|
||||||
mock==1.3.0
|
mock==1.3.0
|
||||||
mockredispy==2.9.0.11
|
mockredispy==2.9.0.11
|
||||||
nose==1.3.7
|
nose==1.3.7
|
||||||
requests-mock==0.7.0
|
requests-mock==0.7.0
|
||||||
freezegun==0.3.7
|
freezegun==0.3.7
|
@ -10,7 +10,7 @@ from setuptools import setup, find_packages
|
|||||||
setup(
|
setup(
|
||||||
name='cartodb_services',
|
name='cartodb_services',
|
||||||
|
|
||||||
version='0.24.0',
|
version='0.24.1',
|
||||||
|
|
||||||
description='CartoDB Services API Python Library',
|
description='CartoDB Services API Python Library',
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user