Merge branch 'development' into 346-user-rate-limits
# Conflicts: # README.md # server/extension/sql/20_geocode_street.sql # server/lib/python/cartodb_services/cartodb_services/metrics/config.py
This commit is contained in:
Javier Goizueta
commit
05b29967c7
18
NEWS.md
18
NEWS.md
@ -1,3 +1,21 @@
|
||||
March 13th, 2017
|
||||
================
|
||||
* Version `0.14.1` of the python library:
|
||||
* Clean up code that reads from non zero padded date keys #206
|
||||
|
||||
March 8th, 2017
|
||||
===============
|
||||
* Version 0.22.0 of the server and version 0.14.0 of the python library
|
||||
* New optional configuration parameters for external services can be provided through `cdb_conf`:
|
||||
- In `heremaps_conf`, under `geocoder.service`: `json_url`, `connect_timeout`, `read_timeout`, `max_retries`, `gen`
|
||||
- In `heremaps_conf`, under `isolines.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries`, `isoline_path`
|
||||
- In `mapzen_conf`, under `geocoder.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries`
|
||||
- In `mapzen_conf`, under `routing.service`: `base_url`, `connect_timeout`, `read_timeout`
|
||||
- In `mapzen_conf`, under `matrix.service`: `one_to_many_url`, `connect_timeout`, `read_timeout`
|
||||
- In `mapzen_conf`, under `isochrones.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries`
|
||||
* Strictly speaking, version 0.14.0 of the python library is not compatible with 0.13.0, but the changes are made on method signatures with default values that were not used from the PG extension.
|
||||
* Improvements to the Mapzen geocoder client, that should yield better results overall. See #342
|
||||
|
||||
February 2st, 2017
|
||||
===================
|
||||
* Version 0.21.0 of the server and version 0.15.0 of the client
|
||||
|
||||
176
README.md
176
README.md
@ -88,15 +88,183 @@ Steps to deploy a new Data Services API version :
|
||||
|
||||
- configure the user DB:
|
||||
|
||||
```sql
|
||||
-- Point to the dataservices server DB (you can use a specific database for the server or your same user's):
|
||||
SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
|
||||
### Server configuration
|
||||
|
||||
Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`.
|
||||
|
||||
#### Redis configuration
|
||||
|
||||
If sentinel is used:
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'redis_metadata_config',
|
||||
'{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'
|
||||
);
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'redis_metrics_config',
|
||||
'{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'
|
||||
);
|
||||
```
|
||||
|
||||
If sentinel is not used:
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'redis_metadata_config',
|
||||
'{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
|
||||
);
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'redis_metrics_config',
|
||||
'{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Users/Organizations
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'user_config',
|
||||
'{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}'
|
||||
);
|
||||
```
|
||||
|
||||
#### HERE configuration
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'heremaps_conf',
|
||||
'{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Mapzen configuration
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapzen_conf',
|
||||
'{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Data Observatory
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'data_observatory_conf',
|
||||
'{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Logger
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'logger_conf',
|
||||
'{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Environment
|
||||
|
||||
The execution environment (development/staging/production) affects rollbar messages and other details.
|
||||
The production environment is used by default.
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'server_conf',
|
||||
'{"environment": "[development|staging|production]"}'
|
||||
);
|
||||
```
|
||||
### Server optional configuration
|
||||
|
||||
External services (Mapzen, Here) can have optional configuration, which is only needed for using non-standard services, such as on-premise installations. We can add the service parameters to an existing configuration like this:
|
||||
|
||||
```
|
||||
# Here geocoder
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'heremaps_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('heremaps_conf')),
|
||||
'{geocoder, service}',
|
||||
'{"json_url":"https://geocoder.api.here.com/6.2/geocode.json","gen":9,"read_timeout":60,"connect_timeout":10,"max_retries":1}'
|
||||
)::json
|
||||
);
|
||||
|
||||
# Here isolines
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'heremaps_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('heremaps_conf')),
|
||||
'{isolines, service}',
|
||||
'{"base_url":"https://isoline.route.api.here.com","isoline_path":"/routing/7.2/calculateisoline.json","read_timeout":60,"connect_timeout":10,"max_retries":1}'
|
||||
)::json
|
||||
);
|
||||
|
||||
# Mapzen geocoder
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapzen_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('mapzen_conf')),
|
||||
'{geocoder, service}',
|
||||
'{"base_url":"https://search.mapzen.com/v1/search","read_timeout":60,"connect_timeout":10,"max_retries":1}'
|
||||
)::json
|
||||
);
|
||||
|
||||
# Mapzen isochrones
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapzen_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('mapzen_conf')),
|
||||
'{isochrones, service}',
|
||||
'{"base_url":"https://matrix.mapzen.com/isochrone","read_timeout":60,"connect_timeout":10,"max_retries":1}'
|
||||
)::json
|
||||
);
|
||||
|
||||
# Mapzen isolines (matrix service)
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapzen_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('mapzen_conf')),
|
||||
'{matrix, service}',
|
||||
'{"base_url":"https://matrix.mapzen.com/one_to_many","read_timeout":60,"connect_timeout":10}'
|
||||
)::json
|
||||
);
|
||||
|
||||
# Mapzen routing
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapzen_conf',
|
||||
jsonb_set(
|
||||
to_jsonb(CDB_Conf_GetConf('mapzen_conf')),
|
||||
'{routing, service}',
|
||||
'{"base_url":"https://valhalla.mapzen.com/route","read_timeout":60,"connect_timeout":10}'
|
||||
)::json
|
||||
);
|
||||
```
|
||||
### User database configuration
|
||||
|
||||
User (client) databases need also some configuration so that the client extension can access the server:
|
||||
#### Users/Organizations
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}');
|
||||
```
|
||||
|
||||
- configure the search path in order to be able to execute the functions without using the schema:
|
||||
#### Dataservices server
|
||||
|
||||
The `geocoder_server_config` (the name is not accurate for historical reasons) entry points
|
||||
to the dataservices server DB (you can use a specific database for the server or your same user's):
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'geocoder_server_config',
|
||||
'{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}'
|
||||
);
|
||||
```
|
||||
#### Search path
|
||||
|
||||
The search path must be configured in order to be able to execute the functions without using the schema:
|
||||
|
||||
```sql
|
||||
ALTER ROLE "<USER_ROLE>" SET search_path="$user", public, cartodb, cdb_dataservices_client;
|
||||
```
|
||||
|
||||
@ -38,7 +38,7 @@ all: $(DATA)
|
||||
.PHONY: release
|
||||
release: $(EXTENSION).control $(SOURCES_DATA)
|
||||
test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
|
||||
mv *.sql old_versions
|
||||
git mv *.sql old_versions
|
||||
$(SED) $(REPLACEMENTS) $(EXTENSION).control
|
||||
cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
|
||||
$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
|
||||
|
||||
292
server/extension/cdb_dataservices_server--0.21.0--0.22.0.sql
Normal file
292
server/extension/cdb_dataservices_server--0.21.0--0.22.0.sql
Normal file
@ -0,0 +1,292 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.22.0'" to load this file. \quit
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
|
||||
from cartodb_services.mapzen.types import polyline_to_linestring
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
quota_service = QuotaService(user_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params)
|
||||
|
||||
if not waypoints or len(waypoints) < 2:
|
||||
logger.info("Empty origin or destination")
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
waypoint_coords = []
|
||||
for waypoint in waypoints:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
|
||||
waypoint_coords.append(Coordinate(lon,lat))
|
||||
|
||||
resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units)
|
||||
if resp and resp.shape:
|
||||
shape_linestring = polyline_to_linestring(resp.shape)
|
||||
if shape_linestring:
|
||||
quota_service.increment_success_service_use()
|
||||
return [shape_linestring, resp.length, resp.duration]
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to calculate mapzen routing')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.here import HereMapsGeocoder
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using here maps')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger
|
||||
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
|
||||
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
|
||||
|
||||
server_config_backend = ServerConfigBackendFactory().get()
|
||||
environment = ServerEnvironmentBuilder(server_config_backend).get()
|
||||
user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
|
||||
org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
|
||||
|
||||
logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
|
||||
logger = Logger(logger_config)
|
||||
|
||||
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
|
||||
|
||||
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
|
||||
|
||||
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3, search_type='address')
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapzen')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient, MapzenIsolines
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params)
|
||||
mapzen_isolines = MapzenIsolines(client, logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
|
||||
isolines[r] = isoline
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
if len(isolines[r]) >= 3:
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
else:
|
||||
multipolygon = None
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get mapzen isolines')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
from cartodb_services.mapzen.types import coordinates_to_polygon
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
|
||||
logger, user_isolines_routing_config.mapzen_isochrones_service_params)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
resp = mapzen_isochrones.isochrone(origin, mode, data_range)
|
||||
|
||||
if resp:
|
||||
result = []
|
||||
for isochrone in resp:
|
||||
result_polygon = coordinates_to_polygon(isochrone.coordinates)
|
||||
if result_polygon:
|
||||
quota_service.increment_success_service_use()
|
||||
result.append([source, isochrone.duration, result_polygon])
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
result.append([source, isochrone.duration, None])
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get mapzen isochrones')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
292
server/extension/cdb_dataservices_server--0.22.0--0.21.0.sql
Normal file
292
server/extension/cdb_dataservices_server--0.22.0--0.21.0.sql
Normal file
@ -0,0 +1,292 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.21.0'" to load this file. \quit
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
|
||||
from cartodb_services.mapzen.types import polyline_to_linestring
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
quota_service = QuotaService(user_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
|
||||
|
||||
if not waypoints or len(waypoints) < 2:
|
||||
logger.info("Empty origin or destination")
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
waypoint_coords = []
|
||||
for waypoint in waypoints:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
|
||||
waypoint_coords.append(Coordinate(lon,lat))
|
||||
|
||||
resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units)
|
||||
if resp and resp.shape:
|
||||
shape_linestring = polyline_to_linestring(resp.shape)
|
||||
if shape_linestring:
|
||||
quota_service.increment_success_service_use()
|
||||
return [shape_linestring, resp.length, resp.duration]
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to calculate mapzen routing')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.here import HereMapsGeocoder
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using here maps')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger
|
||||
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
|
||||
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
|
||||
|
||||
server_config_backend = ServerConfigBackendFactory().get()
|
||||
environment = ServerEnvironmentBuilder(server_config_backend).get()
|
||||
user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
|
||||
org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
|
||||
|
||||
logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
|
||||
logger = Logger(logger_config)
|
||||
|
||||
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
|
||||
|
||||
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
|
||||
|
||||
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3, search_type='address')
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapzen')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient, MapzenIsolines
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
|
||||
mapzen_isolines = MapzenIsolines(client, logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
|
||||
isolines[r] = isoline
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
if len(isolines[r]) >= 3:
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
else:
|
||||
multipolygon = None
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get mapzen isolines')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
from cartodb_services.mapzen.types import coordinates_to_polygon
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
|
||||
logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
resp = mapzen_isochrones.isochrone(origin, mode, data_range)
|
||||
|
||||
if resp:
|
||||
result = []
|
||||
for isochrone in resp:
|
||||
result_polygon = coordinates_to_polygon(isochrone.coordinates)
|
||||
if result_polygon:
|
||||
quota_service.increment_success_service_use()
|
||||
result.append([source, isochrone.duration, result_polygon])
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
result.append([source, isochrone.duration, None])
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get mapzen isochrones')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
2900
server/extension/cdb_dataservices_server--0.22.0.sql
Normal file
2900
server/extension/cdb_dataservices_server--0.22.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices server extension'
|
||||
default_version = '0.21.0'
|
||||
default_version = '0.22.0'
|
||||
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_server
|
||||
|
||||
@ -31,7 +31,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
|
||||
client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params)
|
||||
|
||||
if not waypoints or len(waypoints) < 2:
|
||||
logger.info("Empty origin or destination")
|
||||
|
||||
@ -85,7 +85,7 @@ RETURNS Geometry AS $$
|
||||
service_manager.check()
|
||||
|
||||
try:
|
||||
geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, logger)
|
||||
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
@ -162,7 +162,7 @@ RETURNS Geometry AS $$
|
||||
service_manager.check()
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, logger)
|
||||
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
untry_iso3 = country_to_iso3(country)
|
||||
|
||||
@ -81,7 +81,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
|
||||
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params)
|
||||
mapzen_isolines = MapzenIsolines(client, logger)
|
||||
|
||||
if source:
|
||||
@ -151,7 +151,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
|
||||
try:
|
||||
mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
|
||||
logger)
|
||||
logger, user_isolines_routing_config.mapzen_isochrones_service_params)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
|
||||
@ -52,14 +52,17 @@ class HereMapsGeocoder(Traceable):
|
||||
'strictlanguagemode'
|
||||
] + ADDRESS_PARAMS
|
||||
|
||||
def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS,
|
||||
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
|
||||
def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS):
|
||||
service_params = service_params or {}
|
||||
self.app_id = app_id
|
||||
self.app_code = app_code
|
||||
self._logger = logger
|
||||
self.maxresults = maxresults
|
||||
self.gen = gen
|
||||
self.host = host
|
||||
self.gen = service_params.get('gen', self.DEFAULT_GEN)
|
||||
self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL)
|
||||
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
|
||||
def geocode(self, **kwargs):
|
||||
params = {}
|
||||
@ -92,9 +95,9 @@ class HereMapsGeocoder(Traceable):
|
||||
request_params.update(params)
|
||||
# TODO Extract HTTP client wrapper
|
||||
session = requests.Session()
|
||||
session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES))
|
||||
session.mount(self.host, HTTPAdapter(max_retries=self.max_retries))
|
||||
response = session.get(self.host, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
|
||||
timeout=(self.connect_timeout, self.read_timeout))
|
||||
self.add_response_data(response, self._logger)
|
||||
if response.status_code == requests.codes.ok:
|
||||
return json.loads(response.text)
|
||||
|
||||
@ -30,12 +30,17 @@ class HereMapsRoutingIsoline(Traceable):
|
||||
'quality'
|
||||
]
|
||||
|
||||
def __init__(self, app_id, app_code, logger,
|
||||
base_url=PRODUCTION_ROUTING_BASE_URL):
|
||||
def __init__(self, app_id, app_code, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._app_id = app_id
|
||||
self._app_code = app_code
|
||||
self._logger = logger
|
||||
self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH)
|
||||
base_url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
|
||||
isoline_path = service_params.get('isoline_path', self.ISOLINE_PATH)
|
||||
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
self._url = "{0}{1}".format(base_url, isoline_path)
|
||||
|
||||
def calculate_isodistance(self, source, mode, data_range, options=[]):
|
||||
return self.__calculate_isolines(source, mode, data_range, 'distance',
|
||||
@ -57,9 +62,9 @@ class HereMapsRoutingIsoline(Traceable):
|
||||
parsed_options)
|
||||
# TODO Extract HTTP client wrapper
|
||||
session = requests.Session()
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self.max_retries))
|
||||
response = requests.get(self._url, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
|
||||
timeout=(self.connect_timeout, self.read_timeout))
|
||||
self.add_response_data(response, self._logger)
|
||||
if response.status_code == requests.codes.ok:
|
||||
return self.__parse_isolines_response(response.text)
|
||||
|
||||
@ -17,23 +17,32 @@ class MapzenGeocoder(Traceable):
|
||||
CONNECT_TIMEOUT = 10
|
||||
MAX_RETRIES = 1
|
||||
|
||||
def __init__(self, app_key, logger, base_url=BASE_URL):
|
||||
def __init__(self, app_key, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._app_key = app_key
|
||||
self._url = base_url
|
||||
self._url = service_params.get('base_url', self.BASE_URL)
|
||||
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
self._logger = logger
|
||||
|
||||
@qps_retry(qps=20)
|
||||
def geocode(self, searchtext, city=None, state_province=None,
|
||||
country=None, search_type=None):
|
||||
|
||||
# Remove the search_type if its address from the params sent to mapzen
|
||||
if search_type and search_type.lower() == 'address':
|
||||
search_type = None
|
||||
|
||||
request_params = self._build_requests_parameters(searchtext, city,
|
||||
state_province,
|
||||
country, search_type)
|
||||
try:
|
||||
# TODO Extract HTTP client wrapper
|
||||
session = requests.Session()
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
|
||||
response = session.get(self._url, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
|
||||
timeout=(self._connect_timeout, self._read_timeout))
|
||||
self.add_response_data(response, self._logger)
|
||||
if response.status_code == requests.codes.ok:
|
||||
return self.__parse_response(response.text)
|
||||
|
||||
@ -20,10 +20,15 @@ class MapzenIsochrones:
|
||||
"car": "auto"
|
||||
}
|
||||
|
||||
def __init__(self, app_key, logger, base_url=BASE_URL):
|
||||
def __init__(self, app_key, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._app_key = app_key
|
||||
self._url = base_url
|
||||
self._logger = logger
|
||||
self._url = service_params.get('base_url', self.BASE_URL)
|
||||
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
|
||||
|
||||
@qps_retry(qps=7)
|
||||
def isochrone(self, locations, costing, ranges):
|
||||
@ -32,10 +37,10 @@ class MapzenIsochrones:
|
||||
try:
|
||||
# TODO Extract HTTP client wrapper
|
||||
session = requests.Session()
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
|
||||
response = session.get(self._url, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT,
|
||||
self.READ_TIMEOUT))
|
||||
timeout=(self._connect_timeout,
|
||||
self._read_timeout))
|
||||
|
||||
if response.status_code is requests.codes.ok:
|
||||
return self._parse_response(response)
|
||||
|
||||
@ -23,9 +23,14 @@ class MatrixClient(Traceable):
|
||||
READ_TIMEOUT = 60
|
||||
CONNECT_TIMEOUT = 10
|
||||
|
||||
def __init__(self, matrix_key, logger):
|
||||
def __init__(self, matrix_key, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._matrix_key = matrix_key
|
||||
self._logger = logger
|
||||
self._url = service_params.get('one_to_many_url', self.ONE_TO_MANY_URL)
|
||||
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
|
||||
|
||||
"""Get distances and times to a set of locations.
|
||||
See https://mapzen.com/documentation/matrix/api-reference/
|
||||
@ -44,8 +49,8 @@ class MatrixClient(Traceable):
|
||||
'costing': costing,
|
||||
'api_key': self._matrix_key
|
||||
}
|
||||
response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
|
||||
response = requests.get(self._url, params=request_params,
|
||||
timeout=(self._connect_timeout, self._read_timeout))
|
||||
self.add_response_data(response, self._logger)
|
||||
|
||||
if response.status_code != requests.codes.ok:
|
||||
|
||||
@ -33,10 +33,14 @@ class MapzenRouting(Traceable):
|
||||
METRICS_UNITS = 'kilometers'
|
||||
IMPERIAL_UNITS = 'miles'
|
||||
|
||||
def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL):
|
||||
def __init__(self, app_key, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._app_key = app_key
|
||||
self._url = base_url
|
||||
self._logger = logger
|
||||
self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
|
||||
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
|
||||
@qps_retry
|
||||
def calculate_route_point_to_point(self, waypoints, mode,
|
||||
@ -50,9 +54,9 @@ class MapzenRouting(Traceable):
|
||||
request_params = self.__parse_request_parameters(json_request_params)
|
||||
# TODO Extract HTTP client wrapper
|
||||
session = requests.Session()
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
|
||||
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
|
||||
response = session.get(self._url, params=request_params,
|
||||
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
|
||||
timeout=(self._connect_timeout, self._read_timeout))
|
||||
self.add_response_data(response, self._logger)
|
||||
if response.status_code == requests.codes.ok:
|
||||
return self.__parse_routing_response(response.text)
|
||||
|
||||
@ -154,6 +154,7 @@ class RoutingConfig(ServiceConfig):
|
||||
if not self._routing_provider:
|
||||
self._routing_provider = self.DEFAULT_PROVIDER
|
||||
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
|
||||
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
|
||||
self._set_monthly_quota()
|
||||
self._set_soft_limit()
|
||||
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
||||
@ -171,6 +172,10 @@ class RoutingConfig(ServiceConfig):
|
||||
def mapzen_api_key(self):
|
||||
return self._mapzen_api_key
|
||||
|
||||
@property
|
||||
def mapzen_service_params(self):
|
||||
return self._mapzen_service_params
|
||||
|
||||
@property
|
||||
def monthly_quota(self):
|
||||
return self._monthly_quota
|
||||
@ -229,8 +234,11 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
if self._isolines_provider == self.HEREMAPS_PROVIDER:
|
||||
self._heremaps_app_id = db_config.heremaps_isolines_app_id
|
||||
self._heremaps_app_code = db_config.heremaps_isolines_app_code
|
||||
self._heremaps_service_params = db_config.heremaps_isolines_service_params
|
||||
elif self._isolines_provider == self.MAPZEN_PROVIDER:
|
||||
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
|
||||
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
|
||||
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
@ -263,10 +271,22 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
def heremaps_app_code(self):
|
||||
return self._heremaps_app_code
|
||||
|
||||
@property
|
||||
def heremaps_service_params(self):
|
||||
return self._heremaps_service_params
|
||||
|
||||
@property
|
||||
def mapzen_matrix_api_key(self):
|
||||
return self._mapzen_matrix_api_key
|
||||
|
||||
@property
|
||||
def mapzen_matrix_service_params(self):
|
||||
return self._mapzen_matrix_service_params
|
||||
|
||||
@property
|
||||
def mapzen_isochrones_service_params(self):
|
||||
return self._mapzen_isochrones_service_params
|
||||
|
||||
@property
|
||||
def mapzen_provider(self):
|
||||
return self._isolines_provider == self.MAPZEN_PROVIDER
|
||||
@ -375,6 +395,7 @@ class GeocoderConfig(ServiceConfig):
|
||||
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
|
||||
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
|
||||
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
|
||||
self._heremaps_service_params = db_config.heremaps_geocoder_service_params
|
||||
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
|
||||
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
||||
@ -382,6 +403,7 @@ class GeocoderConfig(ServiceConfig):
|
||||
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
|
||||
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
|
||||
self._cost_per_hit = 0
|
||||
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
|
||||
|
||||
self._rate_limit = self._get_rate_limit('geocoder')
|
||||
|
||||
@ -437,10 +459,18 @@ class GeocoderConfig(ServiceConfig):
|
||||
def heremaps_app_code(self):
|
||||
return self._heremaps_app_code
|
||||
|
||||
@property
|
||||
def heremaps_service_params(self):
|
||||
return self._heremaps_service_params
|
||||
|
||||
@property
|
||||
def mapzen_api_key(self):
|
||||
return self._mapzen_api_key
|
||||
|
||||
@property
|
||||
def mapzen_service_params(self):
|
||||
return self._mapzen_service_params
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
@ -457,6 +487,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
def rate_limit(self):
|
||||
return self._rate_limit
|
||||
|
||||
@property
|
||||
def service(self):
|
||||
return self._service
|
||||
|
||||
class ServicesDBConfig:
|
||||
|
||||
def __init__(self, db_conn, username, orgname):
|
||||
@ -493,8 +527,10 @@ class ServicesDBConfig:
|
||||
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
|
||||
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
|
||||
'geocoder_cost_per_hit']
|
||||
self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
|
||||
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
|
||||
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
|
||||
self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
|
||||
|
||||
def _get_mapzen_config(self):
|
||||
mapzen_conf_json = self._get_conf('mapzen_conf')
|
||||
@ -504,10 +540,14 @@ class ServicesDBConfig:
|
||||
mapzen_conf = json.loads(mapzen_conf_json)
|
||||
self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
|
||||
self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
|
||||
self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
|
||||
self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
|
||||
self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
|
||||
self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
|
||||
self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
|
||||
self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
|
||||
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
|
||||
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
|
||||
|
||||
def _get_data_observatory_config(self):
|
||||
do_conf_json = self._get_conf('data_observatory_conf')
|
||||
@ -547,6 +587,10 @@ class ServicesDBConfig:
|
||||
def heremaps_isolines_app_code(self):
|
||||
return self._heremaps_isolines_app_code
|
||||
|
||||
@property
|
||||
def heremaps_isolines_service_params(self):
|
||||
return self._heremaps_isolines_service_params
|
||||
|
||||
@property
|
||||
def heremaps_geocoder_app_id(self):
|
||||
return self._heremaps_geocoder_app_id
|
||||
@ -559,6 +603,10 @@ class ServicesDBConfig:
|
||||
def heremaps_geocoder_cost_per_hit(self):
|
||||
return self._heremaps_geocoder_cost_per_hit
|
||||
|
||||
@property
|
||||
def heremaps_geocoder_service_params(self):
|
||||
return self._heremaps_geocoder_service_params
|
||||
|
||||
@property
|
||||
def mapzen_matrix_api_key(self):
|
||||
return self._mapzen_matrix_api_key
|
||||
@ -567,6 +615,14 @@ class ServicesDBConfig:
|
||||
def mapzen_matrix_monthly_quota(self):
|
||||
return self._mapzen_matrix_quota
|
||||
|
||||
@property
|
||||
def mapzen_matrix_service_params(self):
|
||||
return self._mapzen_matrix_service_params
|
||||
|
||||
@property
|
||||
def mapzen_isochrones_service_params(self):
|
||||
return self._mapzen_isochrones_service_params
|
||||
|
||||
@property
|
||||
def mapzen_routing_api_key(self):
|
||||
return self._mapzen_routing_api_key
|
||||
@ -575,6 +631,10 @@ class ServicesDBConfig:
|
||||
def mapzen_routing_monthly_quota(self):
|
||||
return self._mapzen_routing_quota
|
||||
|
||||
@property
|
||||
def mapzen_routing_service_params(self):
|
||||
return self._mapzen_routing_service_params
|
||||
|
||||
@property
|
||||
def mapzen_geocoder_api_key(self):
|
||||
return self._mapzen_geocoder_api_key
|
||||
@ -583,6 +643,10 @@ class ServicesDBConfig:
|
||||
def mapzen_geocoder_monthly_quota(self):
|
||||
return self._mapzen_geocoder_quota
|
||||
|
||||
@property
|
||||
def mapzen_geocoder_service_params(self):
|
||||
return self._mapzen_geocoder_service_params
|
||||
|
||||
@property
|
||||
def data_observatory_connection_str(self):
|
||||
return self._data_observatory_connection_str
|
||||
|
||||
@ -118,10 +118,7 @@ class UserMetricsService:
|
||||
for date in self.__generate_date_range(date_from, date_to):
|
||||
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
|
||||
service, metric, date)
|
||||
score = self._redis_connection.zscore(redis_prefix, date.day)
|
||||
aggregated_metric += int(score) if score else 0
|
||||
zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
|
||||
if str(date.day) != zero_padded_day:
|
||||
score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
|
||||
aggregated_metric += int(score) if score else 0
|
||||
|
||||
|
||||
@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object):
|
||||
log_path,
|
||||
mapzen_api_key,
|
||||
username,
|
||||
organization):
|
||||
organization,
|
||||
service_params):
|
||||
self._geocoding_quota = geocoding_quota
|
||||
self._soft_geocoding_limit = soft_geocoding_limit
|
||||
self._period_end_date = period_end_date
|
||||
@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object):
|
||||
self._mapzen_api_key = mapzen_api_key
|
||||
self._username = username
|
||||
self._organization = organization
|
||||
self._service_params = service_params
|
||||
|
||||
# Kind of generic properties. Note which ones are for actually running the
|
||||
# service and which ones are needed for quota stuff.
|
||||
@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object):
|
||||
def organization(self):
|
||||
return self._organization
|
||||
|
||||
@property
|
||||
def service_params(self):
|
||||
return self._service_params
|
||||
|
||||
# TODO: for BW compat, remove
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object):
|
||||
def get(self):
|
||||
mapzen_server_conf = self._server_conf.get('mapzen_conf')
|
||||
mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
|
||||
mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {})
|
||||
|
||||
geocoding_quota = self._get_quota(mapzen_server_conf)
|
||||
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
|
||||
@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object):
|
||||
log_path,
|
||||
mapzen_api_key,
|
||||
self._username,
|
||||
self._orgname)
|
||||
self._orgname,
|
||||
mapzen_service_params)
|
||||
|
||||
def _get_quota(self, mapzen_server_conf):
|
||||
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
|
||||
|
||||
@ -10,7 +10,7 @@ from setuptools import setup, find_packages
|
||||
setup(
|
||||
name='cartodb_services',
|
||||
|
||||
version='0.13.0',
|
||||
version='0.14.1',
|
||||
|
||||
description='CartoDB Services API Python Library',
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ class TestMapzenGeocoderUserConfig(TestCase):
|
||||
self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
|
||||
"mapzen_conf":
|
||||
{"geocoder":
|
||||
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
|
||||
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}}
|
||||
}, "logger_conf": {}})
|
||||
self._username = 'test_user'
|
||||
self._user_key = "rails:users:{0}".format(self._username)
|
||||
@ -81,6 +81,14 @@ class TestMapzenGeocoderUserConfig(TestCase):
|
||||
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
|
||||
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00')
|
||||
|
||||
def test_config_service_values(self):
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
None).get()
|
||||
assert config.service_params == {"base_url":"http://base"}
|
||||
|
||||
class TestMapzenGeocoderOrgConfig(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@ -152,3 +160,11 @@ class TestMapzenGeocoderOrgConfig(TestCase):
|
||||
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00')
|
||||
self._redis_connection.hset(self._org_key, 'geocoding_quota', '200')
|
||||
self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00')
|
||||
|
||||
def test_config_default_service_values(self):
|
||||
config = MapzenGeocoderConfigBuilder(self._server_config,
|
||||
self._user_config,
|
||||
self._org_config,
|
||||
self._username,
|
||||
self._organization).get()
|
||||
assert config.service_params == {}
|
||||
|
||||
@ -145,3 +145,17 @@ class HereMapsGeocoderTestCase(unittest.TestCase):
|
||||
searchtext='Calle amor de dios',
|
||||
city='Cordoba',
|
||||
country='España')
|
||||
|
||||
def test_geocode_with_nonstandard_url(self, req_mock):
|
||||
geocoder = HereMapsGeocoder(None, None, Mock(), { 'json_url': 'http://nonstandard_here_url' })
|
||||
req_mock.register_uri('GET', 'http://nonstandard_here_url', text=self.GOOD_RESPONSE)
|
||||
response = geocoder.geocode(
|
||||
searchtext='Calle amor de dios',
|
||||
city='Cordoba',
|
||||
country='España')
|
||||
|
||||
self.assertEqual(response[0], -5.2794)
|
||||
self.assertEqual(response[1], 37.70246)
|
||||
|
||||
|
||||
|
||||
|
||||
@ -212,3 +212,20 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
|
||||
parsed_url = urlparse(req_mock.request_history[0].url)
|
||||
url_params = parse_qs(parsed_url.query)
|
||||
self.assertEqual(url_params['destination'][0], 'geo!33.0,1.0')
|
||||
|
||||
def test_isodistance_with_nonstandard_url(self, req_mock):
|
||||
base_url = 'http://nonstandard_base'
|
||||
url = "{0}{1}".format(base_url, HereMapsRoutingIsoline.ISOLINE_PATH)
|
||||
routing = HereMapsRoutingIsoline(None, None, Mock(), { 'base_url': base_url })
|
||||
req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE)
|
||||
response = routing.calculate_isodistance('geo!33.0,1.0', 'car',
|
||||
['1000', '2000'])
|
||||
self.assertEqual(len(response), 2)
|
||||
self.assertEqual(response[0]['range'], 1000)
|
||||
self.assertEqual(response[1]['range'], 2000)
|
||||
self.assertEqual(response[0]['geom'], [u'32.9699707,0.9462833',
|
||||
u'32.9699707,0.9458542',
|
||||
u'32.9699707,0.9462833'])
|
||||
self.assertEqual(response[1]['geom'], [u'32.9699707,0.9462833',
|
||||
u'32.9699707,0.9750366',
|
||||
u'32.9699707,0.9462833'])
|
||||
|
||||
@ -109,3 +109,14 @@ class MapzenGeocoderTestCase(unittest.TestCase):
|
||||
self.geocoder.geocode(
|
||||
searchtext='Calle Siempreviva 3, Valladolid',
|
||||
country='ESP')
|
||||
|
||||
def test_geocode_address_with_nonstandard_url(self, req_mock):
|
||||
nonstandard_url = 'http://nonstandardmapzen'
|
||||
req_mock.register_uri('GET', nonstandard_url, text=self.GOOD_RESPONSE)
|
||||
geocoder = MapzenGeocoder('search-XXXXXXX', Mock(), { 'base_url': nonstandard_url })
|
||||
response = geocoder.geocode(
|
||||
searchtext='Calle Siempreviva 3, Valldolid',
|
||||
country='ESP')
|
||||
|
||||
self.assertEqual(response[0], -4.730928)
|
||||
self.assertEqual(response[1], 41.669034)
|
||||
|
||||
@ -52,3 +52,16 @@ class MapzenIsochronesTestCase(unittest.TestCase):
|
||||
with self.assertRaises(ServiceException):
|
||||
self.mapzen_isochrones.isochrone([-41.484375, 28.993727],
|
||||
'walk', [300, 900])
|
||||
|
||||
def test_nonstandard_url(self, req_mock):
|
||||
url = 'http://serviceurl.com'
|
||||
req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE)
|
||||
mapzen_isochrones = MapzenIsochrones('matrix-xxxxx', Mock(), {'base_url': url})
|
||||
|
||||
response = mapzen_isochrones.isochrone([-41.484375, 28.993727],
|
||||
'walk', [300, 900])
|
||||
self.assertEqual(len(response), 2)
|
||||
self.assertEqual(response[0].coordinates, [[-3.702579,40.430893],[-3.702193,40.430122],[-3.702579,40.430893]])
|
||||
self.assertEqual(response[0].duration, 15)
|
||||
self.assertEqual(response[1].coordinates, [[-3.703050,40.424995],[-3.702546,40.424694],[-3.703050,40.424995]])
|
||||
self.assertEqual(response[1].duration, 5)
|
||||
|
||||
@ -142,3 +142,17 @@ class MapzenRoutingTestCase(unittest.TestCase):
|
||||
self.assertEqual(response.length, 1.261)
|
||||
self.assertEqual(response.duration, 913)
|
||||
self.assertEqual(response.shape, self.GOOD_SHAPE_MULTI)
|
||||
|
||||
def test_nonstandard_url(self, req_mock):
|
||||
url = 'http://serviceurl.com'
|
||||
routing = MapzenRouting('api_key', Mock(), {'base_url': url})
|
||||
req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE_SIMPLE)
|
||||
origin = Coordinate('-120.2', '38.5')
|
||||
destination = Coordinate('-126.4', '43.2')
|
||||
waypoints = [origin, destination]
|
||||
response = routing.calculate_route_point_to_point(waypoints,
|
||||
'car')
|
||||
|
||||
self.assertEqual(response.shape, self.GOOD_SHAPE_SIMPLE)
|
||||
self.assertEqual(response.length, 444.59)
|
||||
self.assertEqual(response.duration, 16969)
|
||||
|
||||
@ -90,19 +90,6 @@ class TestUserService(TestCase):
|
||||
self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400)
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
|
||||
|
||||
@freeze_time("2015-06-01")
|
||||
def test_should_account_for_wrongly_stored_non_padded_keys(self):
|
||||
us = self.__build_user_service('test_user')
|
||||
self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
|
||||
|
||||
@freeze_time("2015-06-01")
|
||||
def test_should_sum_amounts_from_both_key_formats(self):
|
||||
us = self.__build_user_service('test_user')
|
||||
self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
|
||||
self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300)
|
||||
assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700
|
||||
|
||||
@freeze_time("2015-06-15")
|
||||
def test_should_not_request_redis_twice_when_unneeded(self):
|
||||
class MockRedisWithCounter(MockRedis):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user