Merge pull request #429 from CartoDB/development
Release server 0.30.0 , client 0.23.0 and python library 0.16.0
This commit is contained in:
Antonio Carlón
GitHub
commit
6edad7b810
@ -158,6 +158,15 @@ SELECT CDB_Conf_SetConf(
|
||||
);
|
||||
```
|
||||
|
||||
#### Mapbox configuration
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'mapbox_conf',
|
||||
'{"routing": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "matrix": {"api_keys": ["your_api_key"], "monthly_quota": 1500000}}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Data Observatory
|
||||
|
||||
```sql
|
||||
|
||||
200
client/cdb_dataservices_client--0.22.0--0.23.0.sql
Executable file
200
client/cdb_dataservices_client--0.22.0--0.23.0.sql
Executable file
@ -0,0 +1,200 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.23.0'" to load this file. \quit
|
||||
|
||||
-- Make sure we have a sane search path to create/update the extension
|
||||
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
DECLARE
|
||||
ret Geometry;
|
||||
username text;
|
||||
orgname text;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
DECLARE
|
||||
|
||||
username text;
|
||||
orgname text;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options);
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
DECLARE
|
||||
|
||||
username text;
|
||||
orgname text;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options);
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
DECLARE
|
||||
ret Geometry;
|
||||
username text;
|
||||
orgname text;
|
||||
_returned_sqlstate TEXT;
|
||||
_message_text TEXT;
|
||||
_pg_exception_context TEXT;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
|
||||
BEGIN
|
||||
SELECT cdb_dataservices_client._cdb_mapbox_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
RETURN ret;
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
DECLARE
|
||||
|
||||
username text;
|
||||
orgname text;
|
||||
_returned_sqlstate TEXT;
|
||||
_message_text TEXT;
|
||||
_pg_exception_context TEXT;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
|
||||
BEGIN
|
||||
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isochrone(username, orgname, source, mode, range, options);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
DECLARE
|
||||
|
||||
username text;
|
||||
orgname text;
|
||||
_returned_sqlstate TEXT;
|
||||
_message_text TEXT;
|
||||
_pg_exception_context TEXT;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
|
||||
BEGIN
|
||||
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_mapbox_isodistance(username, orgname, source, mode, range, options);
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
CONNECT cdb_dataservices_client._server_conn_str();
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_mapbox_geocode_street_point (username, orgname, searchtext, city, state_province, country);
|
||||
|
||||
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
CONNECT cdb_dataservices_client._server_conn_str();
|
||||
|
||||
SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone (username, orgname, source, mode, range, options);
|
||||
|
||||
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_client.isoline AS $$
|
||||
CONNECT cdb_dataservices_client._server_conn_str();
|
||||
|
||||
SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance (username, orgname, source, mode, range, options);
|
||||
|
||||
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapbox_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] ) TO publicuser;
|
||||
18
client/cdb_dataservices_client--0.23.0--0.22.0.sql
Executable file
18
client/cdb_dataservices_client--0.23.0--0.22.0.sql
Executable file
@ -0,0 +1,18 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.22.0'" to load this file. \quit
|
||||
|
||||
-- Make sure we have a sane search path to create/update the extension
|
||||
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapbox_geocode_street_point (searchtext text, city text, state_province text, country text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapbox_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapbox_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point_exception_safe (searchtext text, city text, state_province text, country text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapbox_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);
|
||||
4742
client/cdb_dataservices_client--0.23.0.sql
Normal file
4742
client/cdb_dataservices_client--0.23.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices client API extension'
|
||||
default_version = '0.22.0'
|
||||
default_version = '0.23.0'
|
||||
requires = 'plproxy, cartodb'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_client
|
||||
|
||||
@ -86,6 +86,14 @@
|
||||
- { name: state_province, type: text, default: 'NULL'}
|
||||
- { name: country, type: text, default: 'NULL'}
|
||||
|
||||
- name: cdb_mapbox_geocode_street_point
|
||||
return_type: Geometry
|
||||
params:
|
||||
- { name: searchtext, type: text}
|
||||
- { name: city, type: text, default: 'NULL'}
|
||||
- { name: state_province, type: text, default: 'NULL'}
|
||||
- { name: country, type: text, default: 'NULL'}
|
||||
|
||||
- name: cdb_mapzen_geocode_street_point
|
||||
return_type: Geometry
|
||||
params:
|
||||
@ -114,6 +122,16 @@
|
||||
- { name: range, type: "integer[]" }
|
||||
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
|
||||
|
||||
- name: cdb_mapbox_isochrone
|
||||
return_type: SETOF cdb_dataservices_client.isoline
|
||||
multi_row: true
|
||||
multi_field: true
|
||||
params:
|
||||
- { name: source, type: "geometry(Geometry, 4326)" }
|
||||
- { name: mode, type: text }
|
||||
- { name: range, type: "integer[]" }
|
||||
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
|
||||
|
||||
- name: cdb_mapzen_isochrone
|
||||
return_type: SETOF cdb_dataservices_client.isoline
|
||||
multi_row: true
|
||||
@ -124,6 +142,16 @@
|
||||
- { name: range, type: "integer[]" }
|
||||
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
|
||||
|
||||
- name: cdb_mapbox_isodistance
|
||||
return_type: SETOF cdb_dataservices_client.isoline
|
||||
multi_row: true
|
||||
multi_field: true
|
||||
params:
|
||||
- { name: source, type: "geometry(Geometry, 4326)" }
|
||||
- { name: mode, type: text }
|
||||
- { name: range, type: "integer[]" }
|
||||
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
|
||||
|
||||
- name: cdb_mapzen_isodistance
|
||||
return_type: SETOF cdb_dataservices_client.isoline
|
||||
multi_row: true
|
||||
|
||||
656
server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql
Normal file
656
server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql
Normal file
@ -0,0 +1,656 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.0'" to load this file. \quit
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT)
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxRouting
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools.polyline import polyline_to_linestring
|
||||
from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
if not waypoints or len(waypoints) < 2:
|
||||
service_manager.logger.info("Empty origin or destination")
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
if len(waypoints) > 25:
|
||||
service_manager.logger.info("Too many waypoints (max 25)")
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
waypoint_coords = []
|
||||
for waypoint in waypoints:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
|
||||
waypoint_coords.append(Coordinate(lon,lat))
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
resp = client.directions(waypoint_coords, profile)
|
||||
if resp and resp.shape:
|
||||
shape_linestring = polyline_to_linestring(resp.shape)
|
||||
if shape_linestring:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
return [shape_linestring, resp.length, int(round(resp.duration))]
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to calculate Mapbox routing')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
origin geometry(Point, 4326),
|
||||
destination geometry(Point, 4326),
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_route_with_point', user_routing_config, logger):
|
||||
waypoints = [origin, destination]
|
||||
|
||||
if user_routing_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
elif user_routing_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
else:
|
||||
raise Exception('Requested routing method is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
|
||||
if user_routing_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
elif user_routing_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
else:
|
||||
raise Exception('Requested routing method is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
|
||||
if user_geocoder_config.heremaps_geocoder:
|
||||
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.google_geocoder:
|
||||
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapzen_geocoder:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
# The configuration is retrieved but no checks are performed on it
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.tools.country import country_to_iso3
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3, search_type='address')
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapzen')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from iso3166 import countries
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxGeocoder
|
||||
from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
country_iso3166 = None
|
||||
if country:
|
||||
country_iso3166 = countries.get(country).alpha2.lower()
|
||||
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3166)
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapbox')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
try:
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
try:
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
try:
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = Coordinate(lon,lat)
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
for r in data_range:
|
||||
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
|
||||
isolines[r] = isoline
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
if len(isolines[r]) >= 3:
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
else:
|
||||
multipolygon = None
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get Mapbox isolines')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools.coordinates import coordinates_to_polygon
|
||||
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = Coordinate(lon,lat)
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
|
||||
|
||||
if resp:
|
||||
result = []
|
||||
for isochrone in resp:
|
||||
result_polygon = coordinates_to_polygon(isochrone.coordinates)
|
||||
if result_polygon:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
result.append([source, isochrone.duration, result_polygon])
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
result.append([source, isochrone.duration, None])
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_isolines_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get Mapbox isochrones')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
if user_isolines_config.google_services_user:
|
||||
raise Exception('This service is not available for google service users.')
|
||||
|
||||
with metrics('cb_isodistance', user_isolines_config, logger):
|
||||
if user_isolines_config.heremaps_provider:
|
||||
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
-- mapbox isodistance
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
if user_isolines_config.google_services_user:
|
||||
raise Exception('This service is not available for google service users.')
|
||||
|
||||
with metrics('cb_isochrone', user_isolines_config, logger):
|
||||
if user_isolines_config.heremaps_provider:
|
||||
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
-- mapbox isochrone
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geomvals geomval[],
|
||||
params JSON,
|
||||
merge BOOLEAN DEFAULT True)
|
||||
RETURNS TABLE (
|
||||
id INT,
|
||||
data JSON
|
||||
) AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_obs_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
|
||||
empty_results = len(geomvals) - len(result)
|
||||
if empty_results > 0:
|
||||
quota_service.increment_empty_service_use(empty_results)
|
||||
if result:
|
||||
quota_service.increment_success_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use(len(geomvals))
|
||||
logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetData')
|
||||
finally:
|
||||
quota_service.increment_total_service_use(len(geomvals))
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geomrefs TEXT[],
|
||||
params JSON)
|
||||
RETURNS TABLE (
|
||||
id TEXT,
|
||||
data JSON
|
||||
) AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_obs_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
|
||||
empty_results = len(geomrefs) - len(result)
|
||||
if empty_results > 0:
|
||||
quota_service.increment_empty_service_use(empty_results)
|
||||
if result:
|
||||
quota_service.increment_success_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use(len(geomrefs))
|
||||
exc_info = sys.exc_info()
|
||||
logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
|
||||
logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetData')
|
||||
finally:
|
||||
quota_service.increment_total_service_use(len(geomrefs))
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geom Geometry(Geometry, 4326),
|
||||
params JSON,
|
||||
max_timespan_rank INTEGER DEFAULT NULL,
|
||||
max_score_rank INTEGER DEFAULT NULL,
|
||||
target_geoms INTEGER DEFAULT NULL)
|
||||
RETURNS JSON AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('obs_getmeta', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
|
||||
if result:
|
||||
return result[0]['meta']
|
||||
else:
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetMeta')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
400
server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql
Normal file
400
server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql
Normal file
@ -0,0 +1,400 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.29.0'" to load this file. \quit
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], mode TEXT);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
origin geometry(Point, 4326),
|
||||
destination geometry(Point, 4326),
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_route_with_point', user_routing_config, logger):
|
||||
waypoints = [origin, destination]
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT,
|
||||
options text[] DEFAULT ARRAY[]::text[],
|
||||
units text DEFAULT 'kilometers')
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_routing_config = GD["user_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
|
||||
if user_geocoder_config.heremaps_geocoder:
|
||||
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.google_geocoder:
|
||||
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapzen_geocoder:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT, state_province TEXT, country TEXT);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
country_iso3 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3, search_type='address')
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapzen')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT, state_province TEXT, country TEXT);
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_geocode_namedplace(username text, orgname text, city_name text, admin1_name text, country_name text);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.metrics import QuotaService, metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
|
||||
try:
|
||||
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
|
||||
country_iso3 = None
|
||||
if country_name:
|
||||
country_iso3 = country_to_iso3(country_name)
|
||||
coordinates = geocoder.geocode(searchtext=city_name, city=None,
|
||||
state_province=admin1_name,
|
||||
country=country_iso3, search_type='locality')
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode city point using mapzen')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]);
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_isochrones(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
if user_isolines_config.google_services_user:
|
||||
raise Exception('This service is not available for google service users.')
|
||||
|
||||
with metrics('cb_isodistance', user_isolines_config, logger):
|
||||
if user_isolines_config.heremaps_provider:
|
||||
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
if user_isolines_config.google_services_user:
|
||||
raise Exception('This service is not available for google service users.')
|
||||
|
||||
with metrics('cb_isochrone', user_isolines_config, logger):
|
||||
if user_isolines_config.heremaps_provider:
|
||||
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geomvals geomval[],
|
||||
params JSON,
|
||||
merge BOOLEAN DEFAULT True)
|
||||
RETURNS TABLE (
|
||||
id INT,
|
||||
data JSON
|
||||
) AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_obs_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
|
||||
empty_results = len(geomvals) - len(result)
|
||||
if empty_results > 0:
|
||||
quota_service.increment_empty_service_use(empty_results)
|
||||
if result:
|
||||
quota_service.increment_success_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use(len(geomvals))
|
||||
logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetData')
|
||||
finally:
|
||||
quota_service.increment_total_service_use(len(geomvals))
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geomrefs TEXT[],
|
||||
params JSON)
|
||||
RETURNS TABLE (
|
||||
id TEXT,
|
||||
data JSON
|
||||
) AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_obs_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
|
||||
empty_results = len(geomrefs) - len(result)
|
||||
if empty_results > 0:
|
||||
quota_service.increment_empty_service_use(empty_results)
|
||||
if result:
|
||||
quota_service.increment_success_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use(len(geomrefs))
|
||||
exc_info = sys.exc_info()
|
||||
logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
|
||||
logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetData')
|
||||
finally:
|
||||
quota_service.increment_total_service_use(len(geomrefs))
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
geom Geometry(Geometry, 4326),
|
||||
params JSON,
|
||||
max_timespan_rank INTEGER DEFAULT NULL,
|
||||
max_score_rank INTEGER DEFAULT NULL,
|
||||
target_geoms INTEGER DEFAULT NULL)
|
||||
RETURNS JSON AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_obs_config = GD["user_obs_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('obs_getmeta', user_obs_config, logger):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
|
||||
if result:
|
||||
return result[0]['meta']
|
||||
else:
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to OBS_GetMeta')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
3344
server/extension/cdb_dataservices_server--0.30.0.sql
Normal file
3344
server/extension/cdb_dataservices_server--0.30.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices server extension'
|
||||
default_version = '0.29.0'
|
||||
default_version = '0.30.0'
|
||||
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_server
|
||||
|
||||
@ -4,6 +4,67 @@ CREATE TYPE cdb_dataservices_server.simple_route AS (
|
||||
duration integer
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
waypoints geometry(Point, 4326)[],
|
||||
mode TEXT)
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxRouting
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools.polyline import polyline_to_linestring
|
||||
from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
if not waypoints or len(waypoints) < 2:
|
||||
service_manager.logger.info("Empty origin or destination")
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
if len(waypoints) > 25:
|
||||
service_manager.logger.info("Too many waypoints (max 25)")
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
|
||||
waypoint_coords = []
|
||||
for waypoint in waypoints:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
|
||||
waypoint_coords.append(Coordinate(lon,lat))
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
resp = client.directions(waypoint_coords, profile)
|
||||
if resp and resp.shape:
|
||||
shape_linestring = polyline_to_linestring(resp.shape)
|
||||
if shape_linestring:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
return [shape_linestring, resp.length, int(round(resp.duration))]
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return [None, None, None]
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to calculate Mapbox routing')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
|
||||
@ -9,6 +9,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
@ -19,9 +20,17 @@ RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
|
||||
with metrics('cdb_route_with_point', user_routing_config, logger):
|
||||
waypoints = [origin, destination]
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
|
||||
if user_routing_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
elif user_routing_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
else:
|
||||
raise Exception('Requested routing method is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
|
||||
@ -35,6 +44,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
|
||||
RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
@ -44,7 +54,14 @@ RETURNS cdb_dataservices_server.simple_route AS $$
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
if user_routing_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
elif user_routing_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
|
||||
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
|
||||
else:
|
||||
raise Exception('Requested routing method is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
@ -608,7 +608,7 @@ RETURNS TABLE (
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger):
|
||||
with metrics('obs_getdata', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
|
||||
@ -667,7 +667,7 @@ RETURNS TABLE (
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('obs_getdata', user_obs_config, logger):
|
||||
with metrics('obs_getdata', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
|
||||
@ -724,7 +724,7 @@ RETURNS JSON AS $$
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
with metrics('obs_getmeta', user_obs_config, logger):
|
||||
with metrics('obs_getmeta', user_obs_config, logger, params):
|
||||
try:
|
||||
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
|
||||
result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
|
||||
WHERE pg_type.typname = 'service_type'
|
||||
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
|
||||
CREATE TYPE cdb_dataservices_server.service_type AS ENUM (
|
||||
'isolines',
|
||||
'hires_geocoder',
|
||||
@ -12,7 +14,9 @@ END $$;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_quota_info') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
|
||||
WHERE pg_type.typname = 'service_quota_info'
|
||||
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
|
||||
CREATE TYPE cdb_dataservices_server.service_quota_info AS (
|
||||
service cdb_dataservices_server.service_type,
|
||||
monthly_quota NUMERIC,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
-- Geocodes a street address given a searchtext and a state and/or country
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
@ -21,6 +22,9 @@ RETURNS Geometry AS $$
|
||||
elif user_geocoder_config.mapzen_geocoder:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
@ -70,6 +74,19 @@ RETURNS Geometry AS $$
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
# The configuration is retrieved but no checks are performed on it
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import LegacyServiceManager
|
||||
@ -132,7 +149,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_po
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.tools.country import country_to_iso3
|
||||
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
@ -165,3 +182,43 @@ RETURNS Geometry AS $$
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from iso3166 import countries
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxGeocoder
|
||||
from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
country_iso3166 = None
|
||||
if country:
|
||||
country_iso3166 = countries.get(country).alpha2.lower()
|
||||
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3166)
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using mapbox')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
@ -2,10 +2,18 @@
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
@ -14,10 +22,18 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
@ -26,57 +42,20 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
try:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.types import country_to_iso3
|
||||
from cartodb_services.metrics import QuotaService, metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
raise Exception('You have reached the limit of your quota')
|
||||
|
||||
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
|
||||
try:
|
||||
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
|
||||
country_iso3 = None
|
||||
if country_name:
|
||||
country_iso3 = country_to_iso3(country_name)
|
||||
coordinates = geocoder.geocode(searchtext=city_name, city=None,
|
||||
state_province=admin1_name,
|
||||
country=country_iso3, search_type='locality')
|
||||
if coordinates:
|
||||
quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode city point using mapzen')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
try:
|
||||
street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
except spiexceptions.ExternalRoutineException as e:
|
||||
import sys
|
||||
logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
|
||||
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
|
||||
|
||||
@ -123,6 +123,70 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = Coordinate(lon,lat)
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
for r in data_range:
|
||||
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
|
||||
isolines[r] = isoline
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
if len(isolines[r]) >= 3:
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
else:
|
||||
multipolygon = None
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get Mapbox isolines')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
|
||||
username TEXT,
|
||||
@ -186,3 +250,64 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
|
||||
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools.coordinates import coordinates_to_polygon
|
||||
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
|
||||
service_manager.assert_within_limits()
|
||||
|
||||
try:
|
||||
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = Coordinate(lon,lat)
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
|
||||
|
||||
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
|
||||
|
||||
if resp:
|
||||
result = []
|
||||
for isochrone in resp:
|
||||
result_polygon = coordinates_to_polygon(isochrone.coordinates)
|
||||
if result_polygon:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
result.append([source, isochrone.duration, result_polygon])
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
result.append([source, isochrone.duration, None])
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
service_manager.quota_service.increment_isolines_service_use(len(result))
|
||||
return result
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return []
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to get Mapbox isochrones')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
|
||||
|
||||
@ -21,6 +21,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
@ -53,3 +56,17 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
-- mapbox isodistance
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
@ -21,6 +21,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
elif user_isolines_config.mapzen_provider:
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
elif user_isolines_config.mapbox_provider:
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
else:
|
||||
raise Exception('Requested isolines provider is not available')
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
@ -52,3 +55,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
-- mapbox isochrone
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
|
||||
return result
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
@ -31,6 +31,12 @@ SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
cdb_conf_setconf
|
||||
------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
|
||||
cdb_conf_setconf
|
||||
------------------
|
||||
|
||||
@ -1,18 +1,21 @@
|
||||
-- Check that the public function is callable, even with no data
|
||||
-- It should return NULL
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
------------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
------------------------------
|
||||
|
||||
@ -32,36 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E
|
||||
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
|
||||
-- This should return the point inserted above
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
(1 row)
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
|
||||
WARNING: Error geocoding namedplace using geocode street point, falling back to internal geocoder
|
||||
cdb_geocode_namedplace_point
|
||||
----------------------------------------------------
|
||||
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
|
||||
|
||||
@ -13,6 +13,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localho
|
||||
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
|
||||
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
|
||||
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
|
||||
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
from routing import MapboxRouting, MapboxRoutingResponse
|
||||
from geocoder import MapboxGeocoder
|
||||
from isolines import MapboxIsolines, MapboxIsochronesResponse
|
||||
from matrix_client import MapboxMatrixClient
|
||||
@ -0,0 +1,93 @@
|
||||
'''
|
||||
Python client for the Mapbox Geocoder service.
|
||||
'''
|
||||
|
||||
import json
|
||||
import requests
|
||||
from mapbox import Geocoder
|
||||
from cartodb_services.metrics import Traceable
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
|
||||
GEOCODER_NAME = 'geocoder_name'
|
||||
EPHEMERAL_GEOCODER = 'mapbox.places'
|
||||
PERMANENT_GEOCODER = 'mapbox.places-permanent'
|
||||
DEFAULT_GEOCODER = EPHEMERAL_GEOCODER
|
||||
|
||||
ENTRY_FEATURES = 'features'
|
||||
ENTRY_CENTER = 'center'
|
||||
ENTRY_GEOMETRY = 'geometry'
|
||||
ENTRY_COORDINATES = 'coordinates'
|
||||
ENTRY_TYPE = 'type'
|
||||
TYPE_POINT = 'Point'
|
||||
|
||||
|
||||
class MapboxGeocoder(Traceable):
|
||||
'''
|
||||
Python wrapper for the Mapbox Geocoder service.
|
||||
'''
|
||||
|
||||
def __init__(self, token, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._token = token
|
||||
self._logger = logger
|
||||
self._geocoder_name = service_params.get(GEOCODER_NAME,
|
||||
EPHEMERAL_GEOCODER)
|
||||
self._geocoder = Geocoder(access_token=self._token,
|
||||
name=self._geocoder_name)
|
||||
|
||||
def _parse_geocoder_response(self, response):
|
||||
json_response = json.loads(response)
|
||||
|
||||
if json_response:
|
||||
feature = json_response[ENTRY_FEATURES][0]
|
||||
|
||||
return self._extract_lng_lat_from_feature(feature)
|
||||
else:
|
||||
return []
|
||||
|
||||
def _extract_lng_lat_from_feature(self, feature):
|
||||
geometry = feature[ENTRY_GEOMETRY]
|
||||
if geometry[ENTRY_TYPE] == TYPE_POINT:
|
||||
location = geometry[ENTRY_COORDINATES]
|
||||
else:
|
||||
location = feature[ENTRY_CENTER]
|
||||
|
||||
longitude = location[0]
|
||||
latitude = location[1]
|
||||
return [longitude, latitude]
|
||||
|
||||
@qps_retry(qps=10)
|
||||
def geocode(self, searchtext, city=None, state_province=None,
|
||||
country=None):
|
||||
address = [searchtext]
|
||||
if city:
|
||||
address.append(city)
|
||||
if state_province:
|
||||
address.append(state_province)
|
||||
|
||||
country = [country] if country else None
|
||||
|
||||
try:
|
||||
response = self._geocoder.forward(address=', '.join(address),
|
||||
country=country,
|
||||
limit=1)
|
||||
|
||||
if response.status_code == requests.codes.ok:
|
||||
return self._parse_geocoder_response(response.text)
|
||||
elif response.status_code == requests.codes.bad_request:
|
||||
return []
|
||||
else:
|
||||
raise ServiceException(response.status_code, response)
|
||||
except requests.Timeout as te:
|
||||
# In case of timeout we want to stop the job because the server
|
||||
# could be down
|
||||
self._logger.error('Timeout connecting to Mapbox geocoding server',
|
||||
te)
|
||||
raise ServiceException('Error geocoding {0} using Mapbox'.format(
|
||||
searchtext), None)
|
||||
except requests.ConnectionError as ce:
|
||||
# Don't raise the exception to continue with the geocoding job
|
||||
self._logger.error('Error connecting to Mapbox geocoding server',
|
||||
exception=ce)
|
||||
return []
|
||||
@ -0,0 +1,168 @@
|
||||
'''
|
||||
Python implementation for Mapbox services based isolines.
|
||||
Uses the Mapbox Time Matrix service.
|
||||
'''
|
||||
|
||||
import json
|
||||
from cartodb_services.tools.spherical import (get_angles,
|
||||
calculate_dest_location)
|
||||
from cartodb_services.mapbox.matrix_client import (validate_profile,
|
||||
DEFAULT_PROFILE,
|
||||
PROFILE_WALKING,
|
||||
PROFILE_DRIVING,
|
||||
PROFILE_CYCLING,
|
||||
ENTRY_DURATIONS)
|
||||
|
||||
MAX_SPEEDS = {
|
||||
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
|
||||
PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed
|
||||
PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed
|
||||
}
|
||||
|
||||
DEFAULT_NUM_ANGLES = 24
|
||||
DEFAULT_MAX_ITERS = 5
|
||||
DEFAULT_TOLERANCE = 0.1
|
||||
|
||||
MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES
|
||||
MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS
|
||||
MATRIX_TOLERANCE = DEFAULT_TOLERANCE
|
||||
|
||||
UNIT_FACTOR_ISOCHRONE = 1.0
|
||||
UNIT_FACTOR_ISODISTANCE = 1000.0
|
||||
DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE
|
||||
|
||||
|
||||
class MapboxIsolines():
|
||||
'''
|
||||
Python wrapper for Mapbox services based isolines.
|
||||
'''
|
||||
|
||||
def __init__(self, matrix_client, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._matrix_client = matrix_client
|
||||
self._logger = logger
|
||||
|
||||
def _calculate_matrix_cost(self, origin, targets, isorange,
|
||||
profile=DEFAULT_PROFILE,
|
||||
unit_factor=UNIT_FACTOR_ISOCHRONE,
|
||||
number_of_angles=MATRIX_NUM_ANGLES):
|
||||
response = self._matrix_client.matrix([origin] + targets,
|
||||
profile)
|
||||
json_response = json.loads(response)
|
||||
|
||||
costs = [None] * number_of_angles
|
||||
|
||||
for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
|
||||
if cost:
|
||||
costs[idx] = cost * unit_factor
|
||||
else:
|
||||
costs[idx] = isorange
|
||||
|
||||
return costs
|
||||
|
||||
def calculate_isochrone(self, origin, time_ranges,
|
||||
profile=DEFAULT_PROFILE):
|
||||
validate_profile(profile)
|
||||
|
||||
max_speed = MAX_SPEEDS[profile]
|
||||
|
||||
isochrones = []
|
||||
for time_range in time_ranges:
|
||||
upper_rmax = max_speed * time_range # an upper bound for the radius
|
||||
|
||||
coordinates = self.calculate_isoline(origin=origin,
|
||||
isorange=time_range,
|
||||
upper_rmax=upper_rmax,
|
||||
cost_method=self._calculate_matrix_cost,
|
||||
profile=profile,
|
||||
unit_factor=UNIT_FACTOR_ISOCHRONE,
|
||||
number_of_angles=MATRIX_NUM_ANGLES,
|
||||
max_iterations=MATRIX_MAX_ITERS,
|
||||
tolerance=MATRIX_TOLERANCE)
|
||||
isochrones.append(MapboxIsochronesResponse(coordinates,
|
||||
time_range))
|
||||
return isochrones
|
||||
|
||||
def calculate_isodistance(self, origin, distance_range,
|
||||
profile=DEFAULT_PROFILE):
|
||||
validate_profile(profile)
|
||||
|
||||
max_speed = MAX_SPEEDS[profile]
|
||||
time_range = distance_range / max_speed
|
||||
|
||||
return self.calculate_isochrone(origin=origin,
|
||||
time_ranges=[time_range],
|
||||
profile=profile)[0].coordinates
|
||||
|
||||
def calculate_isoline(self, origin, isorange, upper_rmax,
|
||||
cost_method=_calculate_matrix_cost,
|
||||
profile=DEFAULT_PROFILE,
|
||||
unit_factor=DEFAULT_UNIT_FACTOR,
|
||||
number_of_angles=DEFAULT_NUM_ANGLES,
|
||||
max_iterations=DEFAULT_MAX_ITERS,
|
||||
tolerance=DEFAULT_TOLERANCE):
|
||||
# Formally, a solution is an array of {angle, radius, lat, lon, cost}
|
||||
# with cardinality number_of_angles
|
||||
# we're looking for a solution in which
|
||||
# abs(cost - isorange) / isorange <= TOLERANCE
|
||||
|
||||
# Initial setup
|
||||
angles = get_angles(number_of_angles)
|
||||
rmax = [upper_rmax] * number_of_angles
|
||||
rmin = [0.0] * number_of_angles
|
||||
location_estimates = [calculate_dest_location(origin, a,
|
||||
upper_rmax / 2.0)
|
||||
for a in angles]
|
||||
|
||||
# Iterate to refine the first solution
|
||||
for i in xrange(0, max_iterations):
|
||||
# Calculate the "actual" cost for each location estimate.
|
||||
# NOTE: sometimes it cannot calculate the cost and returns None.
|
||||
# Just assume isorange and stop the calculations there
|
||||
|
||||
costs = cost_method(origin=origin, targets=location_estimates,
|
||||
isorange=isorange, profile=profile,
|
||||
unit_factor=unit_factor,
|
||||
number_of_angles=number_of_angles)
|
||||
|
||||
errors = [(cost - isorange) / float(isorange) for cost in costs]
|
||||
max_abs_error = max([abs(e) for e in errors])
|
||||
if max_abs_error <= tolerance:
|
||||
# good enough, stop there
|
||||
break
|
||||
|
||||
# let's refine the solution, binary search
|
||||
for j in xrange(0, number_of_angles):
|
||||
|
||||
if abs(errors[j]) > tolerance:
|
||||
if errors[j] > 0:
|
||||
rmax[j] = (rmax[j] + rmin[j]) / 2.0
|
||||
else:
|
||||
rmin[j] = (rmax[j] + rmin[j]) / 2.0
|
||||
|
||||
location_estimates[j] = calculate_dest_location(origin,
|
||||
angles[j],
|
||||
(rmax[j] + rmin[j]) / 2.0)
|
||||
|
||||
# delete points that got None
|
||||
location_estimates_filtered = []
|
||||
for i, c in enumerate(costs):
|
||||
if c != isorange:
|
||||
location_estimates_filtered.append(location_estimates[i])
|
||||
|
||||
return location_estimates_filtered
|
||||
|
||||
|
||||
class MapboxIsochronesResponse:
|
||||
|
||||
def __init__(self, coordinates, duration):
|
||||
self._coordinates = coordinates
|
||||
self._duration = duration
|
||||
|
||||
@property
|
||||
def coordinates(self):
|
||||
return self._coordinates
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
return self._duration
|
||||
@ -0,0 +1,88 @@
|
||||
'''
|
||||
Python client for the Mapbox Time Matrix service.
|
||||
'''
|
||||
|
||||
import requests
|
||||
from cartodb_services.metrics import Traceable
|
||||
from cartodb_services.tools.coordinates import (validate_coordinates,
|
||||
marshall_coordinates)
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
|
||||
BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
|
||||
'{coordinates}'
|
||||
'?access_token={token}'
|
||||
'&sources=0' # Set the first coordinate as source...
|
||||
'&destinations=all') # ...and the rest as destinations
|
||||
|
||||
NUM_COORDINATES_MIN = 2 # https://www.mapbox.com/api-documentation/#matrix
|
||||
NUM_COORDINATES_MAX = 25 # https://www.mapbox.com/api-documentation/#matrix
|
||||
|
||||
PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
|
||||
PROFILE_DRIVING = 'driving'
|
||||
PROFILE_CYCLING = 'cycling'
|
||||
PROFILE_WALKING = 'walking'
|
||||
DEFAULT_PROFILE = PROFILE_DRIVING
|
||||
|
||||
VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
|
||||
PROFILE_DRIVING,
|
||||
PROFILE_CYCLING,
|
||||
PROFILE_WALKING]
|
||||
|
||||
ENTRY_DURATIONS = 'durations'
|
||||
|
||||
|
||||
def validate_profile(profile):
|
||||
if profile not in VALID_PROFILES:
|
||||
raise ValueError('{profile} is not a valid profile. '
|
||||
'Valid profiles are: {valid_profiles}'.format(
|
||||
profile=profile,
|
||||
valid_profiles=', '.join(
|
||||
[x for x in VALID_PROFILES])))
|
||||
|
||||
|
||||
class MapboxMatrixClient(Traceable):
|
||||
'''
|
||||
Python wrapper for the Mapbox Time Matrix service.
|
||||
'''
|
||||
|
||||
def __init__(self, token, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._token = token
|
||||
self._logger = logger
|
||||
|
||||
def _uri(self, coordinates, profile=DEFAULT_PROFILE):
|
||||
return BASEURI.format(profile=profile, coordinates=coordinates,
|
||||
token=self._token)
|
||||
|
||||
@qps_retry(qps=1)
|
||||
def matrix(self, coordinates, profile=DEFAULT_PROFILE):
|
||||
validate_profile(profile)
|
||||
validate_coordinates(coordinates,
|
||||
NUM_COORDINATES_MIN, NUM_COORDINATES_MAX)
|
||||
|
||||
coords = marshall_coordinates(coordinates)
|
||||
|
||||
uri = self._uri(coords, profile)
|
||||
|
||||
try:
|
||||
response = requests.get(uri)
|
||||
|
||||
if response.status_code == requests.codes.ok:
|
||||
return response.text
|
||||
elif response.status_code == requests.codes.bad_request:
|
||||
return '{}'
|
||||
else:
|
||||
raise ServiceException(response.status_code, response)
|
||||
except requests.Timeout as te:
|
||||
# In case of timeout we want to stop the job because the server
|
||||
# could be down
|
||||
self._logger.error('Timeout connecting to Mapbox matrix service',
|
||||
te)
|
||||
raise ServiceException('Error getting matrix data from Mapbox',
|
||||
None)
|
||||
except requests.ConnectionError as ce:
|
||||
# Don't raise the exception to continue with the geocoding job
|
||||
self._logger.error('Error connecting to Mapbox matrix service',
|
||||
exception=ce)
|
||||
return '{}'
|
||||
@ -0,0 +1,127 @@
|
||||
'''
|
||||
Python client for the Mapbox Routing service.
|
||||
'''
|
||||
|
||||
import json
|
||||
import requests
|
||||
from cartodb_services.metrics import Traceable
|
||||
from cartodb_services.tools import PolyLine
|
||||
from cartodb_services.tools.coordinates import (validate_coordinates,
|
||||
marshall_coordinates)
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
|
||||
BASEURI = ('https://api.mapbox.com/directions/v5/mapbox/{profile}/'
|
||||
'{coordinates}'
|
||||
'?access_token={token}'
|
||||
'&overview={overview}')
|
||||
|
||||
NUM_WAYPOINTS_MIN = 2 # https://www.mapbox.com/api-documentation/#directions
|
||||
NUM_WAYPOINTS_MAX = 25 # https://www.mapbox.com/api-documentation/#directions
|
||||
|
||||
PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
|
||||
PROFILE_DRIVING = 'driving'
|
||||
PROFILE_CYCLING = 'cycling'
|
||||
PROFILE_WALKING = 'walking'
|
||||
DEFAULT_PROFILE = PROFILE_DRIVING
|
||||
|
||||
DEFAULT_OVERVIEW = 'full'
|
||||
|
||||
VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
|
||||
PROFILE_DRIVING,
|
||||
PROFILE_CYCLING,
|
||||
PROFILE_WALKING]
|
||||
|
||||
ENTRY_ROUTES = 'routes'
|
||||
ENTRY_GEOMETRY = 'geometry'
|
||||
ENTRY_DURATION = 'duration'
|
||||
ENTRY_DISTANCE = 'distance'
|
||||
|
||||
|
||||
class MapboxRouting(Traceable):
|
||||
'''
|
||||
Python wrapper for the Mapbox Routing service.
|
||||
'''
|
||||
|
||||
def __init__(self, token, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._token = token
|
||||
self._logger = logger
|
||||
|
||||
def _uri(self, coordinates, profile=DEFAULT_PROFILE,
|
||||
overview=DEFAULT_OVERVIEW):
|
||||
return BASEURI.format(profile=profile, coordinates=coordinates,
|
||||
token=self._token, overview=overview)
|
||||
|
||||
def _validate_profile(self, profile):
|
||||
if profile not in VALID_PROFILES:
|
||||
raise ValueError('{profile} is not a valid profile. '
|
||||
'Valid profiles are: {valid_profiles}'.format(
|
||||
profile=profile,
|
||||
valid_profiles=', '.join(
|
||||
[x for x in VALID_PROFILES])))
|
||||
|
||||
def _parse_routing_response(self, response):
|
||||
json_response = json.loads(response)
|
||||
|
||||
if json_response:
|
||||
route = json_response[ENTRY_ROUTES][0] # Force the first route
|
||||
|
||||
geometry = PolyLine().decode(route[ENTRY_GEOMETRY])
|
||||
distance = route[ENTRY_DISTANCE]
|
||||
duration = route[ENTRY_DURATION]
|
||||
|
||||
return MapboxRoutingResponse(geometry, distance, duration)
|
||||
else:
|
||||
return MapboxRoutingResponse(None, None, None)
|
||||
|
||||
@qps_retry(qps=1)
|
||||
def directions(self, waypoints, profile=DEFAULT_PROFILE):
|
||||
self._validate_profile(profile)
|
||||
validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX)
|
||||
|
||||
coordinates = marshall_coordinates(waypoints)
|
||||
|
||||
uri = self._uri(coordinates, profile)
|
||||
|
||||
try:
|
||||
response = requests.get(uri)
|
||||
|
||||
if response.status_code == requests.codes.ok:
|
||||
return self._parse_routing_response(response.text)
|
||||
elif response.status_code == requests.codes.bad_request:
|
||||
return MapboxRoutingResponse(None, None, None)
|
||||
else:
|
||||
raise ServiceException(response.status_code, response)
|
||||
except requests.Timeout as te:
|
||||
# In case of timeout we want to stop the job because the server
|
||||
# could be down
|
||||
self._logger.error('Timeout connecting to Mapbox routing service',
|
||||
te)
|
||||
raise ServiceException('Error getting routing data from Mapbox',
|
||||
None)
|
||||
except requests.ConnectionError as ce:
|
||||
# Don't raise the exception to continue with the geocoding job
|
||||
self._logger.error('Error connecting to Mapbox routing service',
|
||||
exception=ce)
|
||||
return MapboxRoutingResponse(None, None, None)
|
||||
|
||||
|
||||
class MapboxRoutingResponse:
|
||||
|
||||
def __init__(self, shape, length, duration):
|
||||
self._shape = shape
|
||||
self._length = length
|
||||
self._duration = duration
|
||||
|
||||
@property
|
||||
def shape(self):
|
||||
return self._shape
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return self._length
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
return self._duration
|
||||
@ -0,0 +1,8 @@
|
||||
MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin'
|
||||
MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin'
|
||||
MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin'
|
||||
|
||||
TRANSPORT_MODE_TO_MAPBOX = {
|
||||
'car': 'driving',
|
||||
'walk': 'walking',
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
|
||||
|
||||
class WrongParams(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr('Wrong parameters passed: ' + json.dumps(self.value))
|
||||
|
||||
|
||||
class MalformedResult(Exception):
|
||||
def __str__(self):
|
||||
return repr('Result structure is malformed')
|
||||
|
||||
|
||||
class TimeoutException(Exception):
|
||||
def __str__(self):
|
||||
return repr('Timeout requesting to mapzen server')
|
||||
|
||||
|
||||
class ServiceException(Exception):
|
||||
def __init__(self, message, response):
|
||||
self.message = message
|
||||
self.response = response
|
||||
|
||||
def response(self):
|
||||
return self.response
|
||||
|
||||
def __str__(self):
|
||||
return self.message
|
||||
@ -3,8 +3,10 @@ import json
|
||||
import re
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from exceptions import WrongParams, MalformedResult, ServiceException
|
||||
from qps import qps_retry
|
||||
from cartodb_services.tools.exceptions import (WrongParams,
|
||||
MalformedResult,
|
||||
ServiceException)
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
from cartodb_services.tools import Coordinate, PolyLine
|
||||
from cartodb_services.metrics import Traceable
|
||||
|
||||
|
||||
@ -3,8 +3,10 @@ import json
|
||||
import re
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from exceptions import WrongParams, MalformedResult, ServiceException
|
||||
from qps import qps_retry
|
||||
from cartodb_services.tools.exceptions import (WrongParams,
|
||||
MalformedResult,
|
||||
ServiceException)
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
|
||||
|
||||
class MapzenIsochrones:
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import requests
|
||||
import json
|
||||
from qps import qps_retry
|
||||
from exceptions import ServiceException
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.metrics import Traceable
|
||||
|
||||
|
||||
|
||||
@ -3,8 +3,10 @@ import json
|
||||
import re
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from exceptions import WrongParams, MalformedResult, ServiceException
|
||||
from qps import qps_retry
|
||||
from cartodb_services.tools.exceptions import (WrongParams,
|
||||
MalformedResult,
|
||||
ServiceException)
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
from cartodb_services.tools import Coordinate, PolyLine
|
||||
from cartodb_services.metrics import MetricsDataGatherer, Traceable
|
||||
|
||||
|
||||
@ -35,18 +35,3 @@ def coordinates_to_polygon(coordinates):
|
||||
geometry = None
|
||||
|
||||
return geometry
|
||||
|
||||
|
||||
def country_to_iso3(country):
|
||||
""" Convert country to its iso3 code """
|
||||
try:
|
||||
country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
|
||||
"'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
|
||||
country_result = plpy.execute(country_plan, [country], 1)
|
||||
if country_result:
|
||||
return country_result[0]['iso3']
|
||||
else:
|
||||
return None
|
||||
except BaseException as e:
|
||||
plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
|
||||
return None
|
||||
|
||||
@ -135,7 +135,8 @@ class RoutingConfig(ServiceConfig):
|
||||
PERIOD_END_DATE = 'period_end_date'
|
||||
ROUTING_PROVIDER_KEY = 'routing_provider'
|
||||
MAPZEN_PROVIDER = 'mapzen'
|
||||
DEFAULT_PROVIDER = 'mapzen'
|
||||
MAPBOX_PROVIDER = 'mapbox'
|
||||
DEFAULT_PROVIDER = MAPZEN_PROVIDER
|
||||
QUOTA_KEY = 'mapzen_routing_quota'
|
||||
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
|
||||
METRICS_LOG_KEY = 'routing_log_path'
|
||||
@ -148,6 +149,8 @@ class RoutingConfig(ServiceConfig):
|
||||
self._routing_provider = self.DEFAULT_PROVIDER
|
||||
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
|
||||
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
|
||||
self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
|
||||
self._mapbox_service_params = self._db_config.mapbox_routing_service_params
|
||||
self._set_monthly_quota()
|
||||
self._set_soft_limit()
|
||||
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
||||
@ -156,11 +159,17 @@ class RoutingConfig(ServiceConfig):
|
||||
def service_type(self):
|
||||
if self._routing_provider == self.MAPZEN_PROVIDER:
|
||||
return 'routing_mapzen'
|
||||
elif self._routing_provider == self.MAPBOX_PROVIDER:
|
||||
return 'routing_mapbox'
|
||||
|
||||
@property
|
||||
def provider(self):
|
||||
return self._routing_provider
|
||||
|
||||
@property
|
||||
def mapzen_provider(self):
|
||||
return self._routing_provider == self.MAPZEN_PROVIDER
|
||||
|
||||
@property
|
||||
def mapzen_api_key(self):
|
||||
return self._mapzen_api_key
|
||||
@ -169,6 +178,18 @@ class RoutingConfig(ServiceConfig):
|
||||
def mapzen_service_params(self):
|
||||
return self._mapzen_service_params
|
||||
|
||||
@property
|
||||
def mapbox_provider(self):
|
||||
return self._routing_provider == self.MAPBOX_PROVIDER
|
||||
|
||||
@property
|
||||
def mapbox_api_keys(self):
|
||||
return self._mapbox_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_service_params(self):
|
||||
return self._mapbox_service_params
|
||||
|
||||
@property
|
||||
def monthly_quota(self):
|
||||
return self._monthly_quota
|
||||
@ -203,8 +224,9 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
ISOLINES_PROVIDER_KEY = 'isolines_provider'
|
||||
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
|
||||
MAPZEN_PROVIDER = 'mapzen'
|
||||
MAPBOX_PROVIDER = 'mapbox'
|
||||
HEREMAPS_PROVIDER = 'heremaps'
|
||||
DEFAULT_PROVIDER = 'mapzen'
|
||||
DEFAULT_PROVIDER = MAPZEN_PROVIDER
|
||||
METRICS_LOG_KEY = 'isolines_log_path'
|
||||
|
||||
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
||||
@ -232,6 +254,10 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
|
||||
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
|
||||
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
|
||||
elif self._isolines_provider == self.MAPBOX_PROVIDER:
|
||||
self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys
|
||||
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
|
||||
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
@ -239,6 +265,8 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
return 'here_isolines'
|
||||
elif self._isolines_provider == self.MAPZEN_PROVIDER:
|
||||
return 'mapzen_isolines'
|
||||
elif self._isolines_provider == self.MAPBOX_PROVIDER:
|
||||
return 'mapbox_isolines'
|
||||
|
||||
@property
|
||||
def google_services_user(self):
|
||||
@ -284,6 +312,22 @@ class IsolinesRoutingConfig(ServiceConfig):
|
||||
def mapzen_provider(self):
|
||||
return self._isolines_provider == self.MAPZEN_PROVIDER
|
||||
|
||||
@property
|
||||
def mapbox_matrix_api_keys(self):
|
||||
return self._mapbox_matrix_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_matrix_service_params(self):
|
||||
return self._mapbox_matrix_service_params
|
||||
|
||||
@property
|
||||
def mapbox_isochrones_service_params(self):
|
||||
return self._mapbox_isochrones_service_params
|
||||
|
||||
@property
|
||||
def mapbox_provider(self):
|
||||
return self._isolines_provider == self.MAPBOX_PROVIDER
|
||||
|
||||
@property
|
||||
def heremaps_provider(self):
|
||||
return self._isolines_provider == self.HEREMAPS_PROVIDER
|
||||
@ -340,12 +384,14 @@ class GeocoderConfig(ServiceConfig):
|
||||
MAPZEN_GEOCODER = 'mapzen'
|
||||
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
|
||||
GEOCODER_PROVIDER = 'geocoder_provider'
|
||||
MAPBOX_GEOCODER = 'mapbox'
|
||||
MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
|
||||
QUOTA_KEY = 'geocoding_quota'
|
||||
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
|
||||
USERNAME_KEY = 'username'
|
||||
ORGNAME_KEY = 'orgname'
|
||||
PERIOD_END_DATE = 'period_end_date'
|
||||
DEFAULT_PROVIDER = 'mapzen'
|
||||
DEFAULT_PROVIDER = MAPZEN_GEOCODER
|
||||
METRICS_LOG_KEY = 'geocoder_log_path'
|
||||
|
||||
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
|
||||
@ -366,6 +412,9 @@ class GeocoderConfig(ServiceConfig):
|
||||
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
|
||||
if not self.mapzen_api_key:
|
||||
raise ConfigException("""Mapzen config is not set up""")
|
||||
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
|
||||
if not self.mapbox_api_keys:
|
||||
raise ConfigException("""Mapbox config is not set up""")
|
||||
|
||||
return True
|
||||
|
||||
@ -397,6 +446,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
|
||||
self._cost_per_hit = 0
|
||||
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
|
||||
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
|
||||
self._mapbox_api_keys = db_config.mapbox_geocoder_api_keys
|
||||
self._cost_per_hit = 0
|
||||
self._mapbox_service_params = db_config.mapbox_geocoder_service_params
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
@ -404,6 +457,8 @@ class GeocoderConfig(ServiceConfig):
|
||||
return 'geocoder_google'
|
||||
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
|
||||
return 'geocoder_mapzen'
|
||||
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
|
||||
return 'geocoder_mapbox'
|
||||
elif self._geocoder_provider == self.NOKIA_GEOCODER:
|
||||
return 'geocoder_here'
|
||||
|
||||
@ -419,6 +474,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
def mapzen_geocoder(self):
|
||||
return self._geocoder_provider == self.MAPZEN_GEOCODER
|
||||
|
||||
@property
|
||||
def mapbox_geocoder(self):
|
||||
return self._geocoder_provider == self.MAPBOX_GEOCODER
|
||||
|
||||
@property
|
||||
def google_client_id(self):
|
||||
return self._google_maps_client_id
|
||||
@ -462,6 +521,14 @@ class GeocoderConfig(ServiceConfig):
|
||||
def mapzen_service_params(self):
|
||||
return self._mapzen_service_params
|
||||
|
||||
@property
|
||||
def mapbox_api_keys(self):
|
||||
return self._mapbox_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_service_params(self):
|
||||
return self._mapbox_service_params
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
@ -478,6 +545,7 @@ class GeocoderConfig(ServiceConfig):
|
||||
def service(self):
|
||||
return self._service
|
||||
|
||||
|
||||
class ServicesDBConfig:
|
||||
|
||||
def __init__(self, db_conn, username, orgname):
|
||||
@ -490,6 +558,7 @@ class ServicesDBConfig:
|
||||
self._get_server_config()
|
||||
self._get_here_config()
|
||||
self._get_mapzen_config()
|
||||
self._get_mapbox_config()
|
||||
self._get_data_observatory_config()
|
||||
|
||||
def _get_server_config(self):
|
||||
@ -535,6 +604,23 @@ class ServicesDBConfig:
|
||||
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
|
||||
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
|
||||
|
||||
def _get_mapbox_config(self):
|
||||
mapbox_conf_json = self._get_conf('mapbox_conf')
|
||||
if not mapbox_conf_json:
|
||||
raise ConfigException('Mapbox configuration missing')
|
||||
else:
|
||||
mapbox_conf = json.loads(mapbox_conf_json)
|
||||
self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
|
||||
self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
|
||||
self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
|
||||
self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
|
||||
self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
|
||||
self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
|
||||
self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
|
||||
self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys']
|
||||
self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
|
||||
self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
|
||||
|
||||
def _get_data_observatory_config(self):
|
||||
do_conf_json = self._get_conf('data_observatory_conf')
|
||||
if not do_conf_json:
|
||||
@ -548,7 +634,6 @@ class ServicesDBConfig:
|
||||
else:
|
||||
self._data_observatory_connection_str = do_conf['connection']['production']
|
||||
|
||||
|
||||
def _get_conf(self, key):
|
||||
try:
|
||||
sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
|
||||
@ -629,6 +714,46 @@ class ServicesDBConfig:
|
||||
def mapzen_geocoder_service_params(self):
|
||||
return self._mapzen_geocoder_service_params
|
||||
|
||||
@property
|
||||
def mapbox_matrix_api_keys(self):
|
||||
return self._mapbox_matrix_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_matrix_monthly_quota(self):
|
||||
return self._mapbox_matrix_quota
|
||||
|
||||
@property
|
||||
def mapbox_matrix_service_params(self):
|
||||
return self._mapbox_matrix_service_params
|
||||
|
||||
@property
|
||||
def mapbox_isochrones_service_params(self):
|
||||
return self._mapbox_isochrones_service_params
|
||||
|
||||
@property
|
||||
def mapbox_routing_api_keys(self):
|
||||
return self._mapbox_routing_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_routing_monthly_quota(self):
|
||||
return self._mapbox_routing_quota
|
||||
|
||||
@property
|
||||
def mapbox_routing_service_params(self):
|
||||
return self._mapbox_routing_service_params
|
||||
|
||||
@property
|
||||
def mapbox_geocoder_api_keys(self):
|
||||
return self._mapbox_geocoder_api_keys
|
||||
|
||||
@property
|
||||
def mapbox_geocoder_monthly_quota(self):
|
||||
return self._mapbox_geocoder_quota
|
||||
|
||||
@property
|
||||
def mapbox_geocoder_service_params(self):
|
||||
return self._mapbox_geocoder_service_params
|
||||
|
||||
@property
|
||||
def data_observatory_connection_str(self):
|
||||
return self._data_observatory_connection_str
|
||||
|
||||
@ -78,9 +78,15 @@ class QuotaChecker:
|
||||
elif re.match('mapzen_isolines',
|
||||
self._user_service_config.service_type) is not None:
|
||||
return self.__check_isolines_quota()
|
||||
elif re.match('mapbox_isolines',
|
||||
self._user_service_config.service_type) is not None:
|
||||
return self.__check_isolines_quota()
|
||||
elif re.match('routing_mapzen',
|
||||
self._user_service_config.service_type) is not None:
|
||||
return self.__check_routing_quota()
|
||||
elif re.match('routing_mapbox',
|
||||
self._user_service_config.service_type) is not None:
|
||||
return self.__check_routing_quota()
|
||||
elif re.match('obs_*',
|
||||
self._user_service_config.service_type) is not None:
|
||||
return self.__check_data_observatory_quota()
|
||||
|
||||
@ -21,7 +21,9 @@ class UserMetricsService:
|
||||
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
|
||||
SERVICE_HERE_ISOLINES = 'here_isolines'
|
||||
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
|
||||
SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
|
||||
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
|
||||
SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
|
||||
SERVICE_OBSERVATORY = 'obs_general'
|
||||
DAY_OF_MONTH_ZERO_PADDED = '%d'
|
||||
|
||||
@ -32,11 +34,12 @@ class UserMetricsService:
|
||||
self._orgname = user_geocoder_config.organization
|
||||
|
||||
def used_quota(self, service_type, date):
|
||||
if service_type == self.SERVICE_HERE_ISOLINES:
|
||||
if service_type in [self.SERVICE_HERE_ISOLINES,
|
||||
self.SERVICE_MAPZEN_ISOLINES,
|
||||
self.SERVICE_MAPBOX_ISOLINES]:
|
||||
return self.__used_isolines_quota(service_type, date)
|
||||
if service_type == self.SERVICE_MAPZEN_ISOLINES:
|
||||
return self.__used_isolines_quota(service_type, date)
|
||||
elif service_type == self.SERVICE_MAPZEN_ROUTING:
|
||||
elif service_type in [self.SERVICE_MAPZEN_ROUTING,
|
||||
self.SERVICE_MAPBOX_ROUTING]:
|
||||
return self.__used_routing_quota(service_type, date)
|
||||
elif service_type == self.SERVICE_OBSERVATORY:
|
||||
return self.__used_observatory_quota(service_type, date)
|
||||
|
||||
@ -0,0 +1,128 @@
|
||||
from dateutil.parser import parse as date_parse
|
||||
from cartodb_services.refactor.service.utils import round_robin
|
||||
from cartodb_services.mapbox.types import MAPBOX_GEOCODER_APIKEY_ROUNDROBIN
|
||||
|
||||
|
||||
class MapboxGeocoderConfig(object):
|
||||
"""
|
||||
Configuration needed to operate the Mapbox geocoder service.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
geocoding_quota,
|
||||
soft_geocoding_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
username,
|
||||
organization,
|
||||
service_params,
|
||||
GD):
|
||||
self._geocoding_quota = geocoding_quota
|
||||
self._soft_geocoding_limit = soft_geocoding_limit
|
||||
self._period_end_date = period_end_date
|
||||
self._cost_per_hit = cost_per_hit
|
||||
self._log_path = log_path
|
||||
self._mapbox_api_keys = mapbox_api_keys
|
||||
self._username = username
|
||||
self._organization = organization
|
||||
self._service_params = service_params
|
||||
self._GD = GD
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
return 'geocoder_mapbox'
|
||||
|
||||
@property
|
||||
def provider(self):
|
||||
return 'mapbox'
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def geocoding_quota(self):
|
||||
return self._geocoding_quota
|
||||
|
||||
@property
|
||||
def soft_geocoding_limit(self):
|
||||
return self._soft_geocoding_limit
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@property
|
||||
def cost_per_hit(self):
|
||||
return self._cost_per_hit
|
||||
|
||||
@property
|
||||
def log_path(self):
|
||||
return self._log_path
|
||||
|
||||
@property
|
||||
def mapbox_api_key(self):
|
||||
return round_robin(self._mapbox_api_keys, self._GD,
|
||||
MAPBOX_GEOCODER_APIKEY_ROUNDROBIN)
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._organization
|
||||
|
||||
@property
|
||||
def service_params(self):
|
||||
return self._service_params
|
||||
|
||||
# TODO: for BW compat, remove
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
return False
|
||||
|
||||
|
||||
class MapboxGeocoderConfigBuilder(object):
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
self._GD = GD
|
||||
|
||||
def get(self):
|
||||
mapbox_server_conf = self._server_conf.get('mapbox_conf')
|
||||
mapbox_api_keys = mapbox_server_conf['geocoder']['api_keys']
|
||||
mapbox_service_params = mapbox_server_conf['geocoder'].get('service', {})
|
||||
|
||||
geocoding_quota = self._get_quota()
|
||||
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
|
||||
cost_per_hit = 0
|
||||
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
|
||||
period_end_date = date_parse(period_end_date_str)
|
||||
|
||||
logger_conf = self._server_conf.get('logger_conf')
|
||||
log_path = logger_conf.get('geocoder_log_path', None)
|
||||
|
||||
return MapboxGeocoderConfig(geocoding_quota,
|
||||
soft_geocoding_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
self._username,
|
||||
self._orgname,
|
||||
mapbox_service_params,
|
||||
self._GD)
|
||||
|
||||
def _get_quota(self):
|
||||
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
|
||||
if geocoding_quota is '':
|
||||
return 0
|
||||
|
||||
return int(geocoding_quota)
|
||||
@ -0,0 +1,123 @@
|
||||
from dateutil.parser import parse as date_parse
|
||||
from cartodb_services.refactor.service.utils import round_robin
|
||||
from cartodb_services.mapbox.types import MAPBOX_ISOLINES_APIKEY_ROUNDROBIN
|
||||
|
||||
|
||||
class MapboxIsolinesConfig(object):
|
||||
"""
|
||||
Configuration needed to operate the Mapbox directions service.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
isolines_quota,
|
||||
soft_isolines_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
username,
|
||||
organization,
|
||||
service_params,
|
||||
GD):
|
||||
self._isolines_quota = isolines_quota
|
||||
self._soft_isolines_limit = soft_isolines_limit
|
||||
self._period_end_date = period_end_date
|
||||
self._cost_per_hit = cost_per_hit
|
||||
self._log_path = log_path
|
||||
self._mapbox_api_keys = mapbox_api_keys
|
||||
self._username = username
|
||||
self._organization = organization
|
||||
self._service_params = service_params
|
||||
self._GD = GD
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
return 'mapbox_isolines'
|
||||
|
||||
@property
|
||||
def provider(self):
|
||||
return 'mapbox'
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def isolines_quota(self):
|
||||
return self._isolines_quota
|
||||
|
||||
@property
|
||||
def soft_isolines_limit(self):
|
||||
return self._soft_isolines_limit
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@property
|
||||
def cost_per_hit(self):
|
||||
return self._cost_per_hit
|
||||
|
||||
@property
|
||||
def log_path(self):
|
||||
return self._log_path
|
||||
|
||||
@property
|
||||
def mapbox_api_key(self):
|
||||
return round_robin(self._mapbox_api_keys, self._GD,
|
||||
MAPBOX_ISOLINES_APIKEY_ROUNDROBIN)
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._organization
|
||||
|
||||
@property
|
||||
def service_params(self):
|
||||
return self._service_params
|
||||
|
||||
|
||||
class MapboxIsolinesConfigBuilder(object):
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
self._GD = GD
|
||||
|
||||
def get(self):
|
||||
mapbox_server_conf = self._server_conf.get('mapbox_conf')
|
||||
mapbox_api_keys = mapbox_server_conf['matrix']['api_keys']
|
||||
mapbox_service_params = mapbox_server_conf['matrix'].get('service', {})
|
||||
|
||||
isolines_quota = self._get_quota()
|
||||
soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true'
|
||||
cost_per_hit = 0
|
||||
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
|
||||
period_end_date = date_parse(period_end_date_str)
|
||||
|
||||
logger_conf = self._server_conf.get('logger_conf')
|
||||
log_path = logger_conf.get('isolines_log_path', None)
|
||||
|
||||
return MapboxIsolinesConfig(isolines_quota,
|
||||
soft_isolines_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
self._username,
|
||||
self._orgname,
|
||||
mapbox_service_params,
|
||||
self._GD)
|
||||
|
||||
def _get_quota(self):
|
||||
isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota')
|
||||
if isolines_quota is '':
|
||||
return 0
|
||||
|
||||
return int(isolines_quota)
|
||||
@ -0,0 +1,131 @@
|
||||
from dateutil.parser import parse as date_parse
|
||||
from cartodb_services.refactor.service.utils import round_robin
|
||||
from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDROBIN
|
||||
|
||||
|
||||
class MapboxRoutingConfig(object):
|
||||
"""
|
||||
Configuration needed to operate the Mapbox directions service.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
routing_quota,
|
||||
soft_routing_limit,
|
||||
monthly_quota,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
username,
|
||||
organization,
|
||||
service_params,
|
||||
GD):
|
||||
self._routing_quota = routing_quota
|
||||
self._soft_routing_limit = soft_routing_limit
|
||||
self._monthly_quota = monthly_quota
|
||||
self._period_end_date = period_end_date
|
||||
self._cost_per_hit = cost_per_hit
|
||||
self._log_path = log_path
|
||||
self._mapbox_api_keys = mapbox_api_keys
|
||||
self._username = username
|
||||
self._organization = organization
|
||||
self._service_params = service_params
|
||||
self._GD = GD
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
return 'routing_mapbox'
|
||||
|
||||
@property
|
||||
def provider(self):
|
||||
return 'mapbox'
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def routing_quota(self):
|
||||
return self._routing_quota
|
||||
|
||||
@property
|
||||
def soft_limit(self):
|
||||
return self._soft_routing_limit
|
||||
|
||||
@property
|
||||
def monthly_quota(self):
|
||||
return self._monthly_quota
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@property
|
||||
def cost_per_hit(self):
|
||||
return self._cost_per_hit
|
||||
|
||||
@property
|
||||
def log_path(self):
|
||||
return self._log_path
|
||||
|
||||
@property
|
||||
def mapbox_api_key(self):
|
||||
return round_robin(self._mapbox_api_keys, self._GD,
|
||||
MAPBOX_ROUTING_APIKEY_ROUNDROBIN)
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._organization
|
||||
|
||||
@property
|
||||
def service_params(self):
|
||||
return self._service_params
|
||||
|
||||
|
||||
class MapboxRoutingConfigBuilder(object):
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
self._GD = GD
|
||||
|
||||
def get(self):
|
||||
mapbox_server_conf = self._server_conf.get('mapbox_conf')
|
||||
mapbox_api_keys = mapbox_server_conf['routing']['api_keys']
|
||||
monthly_quota = mapbox_server_conf['routing']['monthly_quota']
|
||||
mapbox_service_params = mapbox_server_conf['routing'].get('service', {})
|
||||
|
||||
routing_quota = self._get_quota()
|
||||
soft_routing_limit = self._user_conf.get('soft_mapzen_routing_limit').lower() == 'true'
|
||||
cost_per_hit = 0
|
||||
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
|
||||
period_end_date = date_parse(period_end_date_str)
|
||||
|
||||
logger_conf = self._server_conf.get('logger_conf')
|
||||
log_path = logger_conf.get('routing_log_path', None)
|
||||
|
||||
return MapboxRoutingConfig(routing_quota,
|
||||
soft_routing_limit,
|
||||
monthly_quota,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
mapbox_api_keys,
|
||||
self._username,
|
||||
self._orgname,
|
||||
mapbox_service_params,
|
||||
self._GD)
|
||||
|
||||
def _get_quota(self):
|
||||
routing_quota = self._org_conf.get('mapzen_routing_quota') or self._user_conf.get('mapzen_routing_quota')
|
||||
if routing_quota is '':
|
||||
return 0
|
||||
|
||||
return int(routing_quota)
|
||||
@ -86,7 +86,7 @@ class MapzenGeocoderConfig(object):
|
||||
|
||||
class MapzenGeocoderConfigBuilder(object):
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname):
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD=None):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
def round_robin(elements, GD, key):
|
||||
rr = GD[key] if key in GD else 0
|
||||
value = elements[rr]
|
||||
GD[key] = rr + 1 if rr < len(elements) - 1 else 0
|
||||
|
||||
return value
|
||||
@ -4,4 +4,4 @@ from polyline import PolyLine
|
||||
from log import Logger, LoggerConfig
|
||||
from rate_limiter import RateLimiter
|
||||
from service_manager import ServiceManager, RateLimitExceeded
|
||||
from legacy_service_manager import LegacyServiceManager
|
||||
from legacy_service_manager import LegacyServiceManager
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
import plpy
|
||||
|
||||
|
||||
class Coordinate:
|
||||
"""Class that represents a generic form of coordinates to be used
|
||||
by the services """
|
||||
@ -17,5 +20,43 @@ class Coordinate:
|
||||
def to_json(self):
|
||||
return "{{\"lon\": {0},\"lat\": {1}}}".format(self._longitude,
|
||||
self._latitude)
|
||||
|
||||
def __str__(self):
|
||||
return "{0}, {1}".format(self._longitude, self._latitude)
|
||||
return "{0}, {1}".format(self._longitude, self._latitude)
|
||||
|
||||
|
||||
def validate_coordinates(coordinates,
|
||||
num_coordinates_min, num_coordinates_max):
|
||||
if not coordinates:
|
||||
raise ValueError('Invalid (empty) coordinates.')
|
||||
|
||||
if len(coordinates) < num_coordinates_min \
|
||||
or len(coordinates) > num_coordinates_max:
|
||||
raise ValueError('Invalid number of coordinates. '
|
||||
'Must be between {min} and {max}'.format(
|
||||
min=num_coordinates_min,
|
||||
max=num_coordinates_max))
|
||||
|
||||
|
||||
def marshall_coordinates(coordinates):
|
||||
return ';'.join([str(coordinate).replace(' ', '')
|
||||
for coordinate in coordinates])
|
||||
|
||||
|
||||
def coordinates_to_polygon(coordinates):
|
||||
"""Convert a Coordinate array coordinates to a PostGIS polygon"""
|
||||
coordinates.append(coordinates[0]) # Close the ring
|
||||
result_coordinates = []
|
||||
for coordinate in coordinates:
|
||||
result_coordinates.append("%s %s" % (coordinate.longitude,
|
||||
coordinate.latitude))
|
||||
wkt_coordinates = ','.join(result_coordinates)
|
||||
|
||||
try:
|
||||
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates)
|
||||
geometry = plpy.execute(sql, 1)[0]['geom']
|
||||
except BaseException as e:
|
||||
plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e))
|
||||
geometry = None
|
||||
|
||||
return geometry
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
import plpy
|
||||
|
||||
|
||||
def country_to_iso3(country):
|
||||
""" Convert country to its iso3 code """
|
||||
if not country:
|
||||
return None
|
||||
|
||||
try:
|
||||
country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
|
||||
"'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
|
||||
country_result = plpy.execute(country_plan, [country], 1)
|
||||
if country_result:
|
||||
return country_result[0]['iso3']
|
||||
else:
|
||||
return None
|
||||
except BaseException as e:
|
||||
plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
|
||||
return None
|
||||
@ -0,0 +1,31 @@
|
||||
import json
|
||||
|
||||
|
||||
class TimeoutException(Exception):
|
||||
def __str__(self):
|
||||
return repr('Timeout requesting to server')
|
||||
|
||||
|
||||
class ServiceException(Exception):
|
||||
def __init__(self, message, response):
|
||||
self.message = message
|
||||
self.response = response
|
||||
|
||||
def response(self):
|
||||
return self.response
|
||||
|
||||
def __str__(self):
|
||||
return self.message
|
||||
|
||||
|
||||
class WrongParams(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr('Wrong parameters passed: ' + json.dumps(self.value))
|
||||
|
||||
|
||||
class MalformedResult(Exception):
|
||||
def __str__(self):
|
||||
return repr('Result structure is malformed')
|
||||
@ -1,5 +1,4 @@
|
||||
from itertools import tee, izip
|
||||
from math import trunc
|
||||
import plpy
|
||||
|
||||
|
||||
class PolyLine:
|
||||
@ -46,3 +45,22 @@ class PolyLine:
|
||||
coordinate |= c << (i * 5)
|
||||
|
||||
return coordinate
|
||||
|
||||
|
||||
def polyline_to_linestring(polyline):
|
||||
"""Convert a Mapzen polyline shape to a PostGIS linestring"""
|
||||
coordinates = []
|
||||
for point in polyline:
|
||||
# Divide by 10 because mapzen uses one more decimal than the
|
||||
# google standard (https://mapzen.com/documentation/turn-by-turn/decoding/)
|
||||
coordinates.append("%s %s" % (point[1]/10, point[0]/10))
|
||||
wkt_coordinates = ','.join(coordinates)
|
||||
|
||||
try:
|
||||
sql = "SELECT ST_GeomFromText('LINESTRING({0})', 4326) as geom".format(wkt_coordinates)
|
||||
geometry = plpy.execute(sql, 1)[0]['geom']
|
||||
except BaseException as e:
|
||||
plpy.warning("Can't generate LINESTRING from polyline: {0}".format(e))
|
||||
geometry = None
|
||||
|
||||
return geometry
|
||||
|
||||
@ -6,7 +6,8 @@ from exceptions import TimeoutException
|
||||
DEFAULT_RETRY_TIMEOUT = 60
|
||||
DEFAULT_QUERIES_PER_SECOND = 10
|
||||
|
||||
def qps_retry(original_function=None,**options):
|
||||
|
||||
def qps_retry(original_function=None, **options):
|
||||
""" Query Per Second retry decorator
|
||||
The intention of this decorator is to retry requests against third
|
||||
party services that has QPS restriction.
|
||||
@ -55,7 +56,7 @@ class QPSService:
|
||||
|
||||
def retry(self, first_request_time, retry_count):
|
||||
elapsed = datetime.now() - first_request_time
|
||||
if elapsed.total_seconds() > self._retry_timeout:
|
||||
if elapsed.total_seconds() > self._retry_timeout:
|
||||
raise TimeoutException()
|
||||
|
||||
# inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
|
||||
@ -5,10 +5,12 @@ from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
|
||||
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
|
||||
from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
|
||||
|
||||
|
||||
class RateLimitExceeded(Exception):
|
||||
def __str__(self):
|
||||
return repr('Rate limit exceeded')
|
||||
|
||||
|
||||
class ServiceManagerBase:
|
||||
"""
|
||||
A Service manager collects the configuration needed to use a service,
|
||||
@ -48,6 +50,8 @@ class ServiceManagerBase:
|
||||
@property
|
||||
def logger(self):
|
||||
return self.logger
|
||||
|
||||
|
||||
class ServiceManager(ServiceManagerBase):
|
||||
"""
|
||||
This service manager delegates the configuration parameter details,
|
||||
@ -55,13 +59,13 @@ class ServiceManager(ServiceManagerBase):
|
||||
It uses the refactored configuration classes.
|
||||
"""
|
||||
|
||||
def __init__(self, service, config_builder, username, orgname):
|
||||
def __init__(self, service, config_builder, username, orgname, GD=None):
|
||||
service_config = ServiceConfiguration(service, username, orgname)
|
||||
|
||||
logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get()
|
||||
self.logger = Logger(logger_config)
|
||||
|
||||
self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get()
|
||||
self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname, GD).get()
|
||||
rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
|
||||
|
||||
redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get()
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
from math import cos, sin, pi, radians, degrees, asin, atan2
|
||||
from cartodb_services.tools import Coordinate
|
||||
|
||||
EARTH_RADIUS_METERS = 6367444
|
||||
|
||||
|
||||
def get_angles(number_of_angles): # Angle in radians
|
||||
step = (2.0 * pi) / number_of_angles
|
||||
return [(x * step) for x in xrange(0, number_of_angles)]
|
||||
|
||||
def calculate_dest_location(origin, angle, radius): # Angle in radians
|
||||
origin_lat_radians = radians(origin.latitude)
|
||||
origin_long_radians = radians(origin.longitude)
|
||||
dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / EARTH_RADIUS_METERS) * cos(angle))
|
||||
dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians))
|
||||
|
||||
return Coordinate(degrees(dest_lng_radians), degrees(dest_lat_radians))
|
||||
@ -6,6 +6,7 @@ rollbar==0.13.2
|
||||
# Dependency for googlemaps package
|
||||
requests==2.9.1
|
||||
rratelimit==0.0.4
|
||||
mapbox==0.14.0
|
||||
|
||||
# Test
|
||||
mock==1.3.0
|
||||
|
||||
@ -10,7 +10,7 @@ from setuptools import setup, find_packages
|
||||
setup(
|
||||
name='cartodb_services',
|
||||
|
||||
version='0.15.7',
|
||||
version='0.16.0',
|
||||
|
||||
description='CartoDB Services API Python Library',
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
|
||||
|
||||
class TestGeocoderUserConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
@ -25,6 +25,9 @@ class TestGeocoderUserConfig(TestCase):
|
||||
elif geocoder_provider == 'mapzen':
|
||||
assert geocoder_config.mapzen_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'mapbox':
|
||||
assert geocoder_config.mapbox_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
@ -75,9 +78,10 @@ class TestGeocoderUserConfig(TestCase):
|
||||
'test_user', None)
|
||||
assert geocoder_config.soft_geocoding_limit == False
|
||||
|
||||
|
||||
class TestGeocoderOrgConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
@ -100,6 +104,9 @@ class TestGeocoderOrgConfig(TestCase):
|
||||
elif geocoder_provider == 'mapzen':
|
||||
assert geocoder_config.mapzen_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'mapbox':
|
||||
assert geocoder_config.mapbox_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
@ -160,8 +167,7 @@ class TestGeocoderOrgConfig(TestCase):
|
||||
|
||||
|
||||
class TestIsolinesUserConfig(TestCase):
|
||||
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
@ -175,6 +181,8 @@ class TestIsolinesUserConfig(TestCase):
|
||||
'test_user')
|
||||
if isolines_provider is 'mapzen':
|
||||
assert isolines_config.service_type is 'mapzen_isolines'
|
||||
elif isolines_provider is 'mapbox':
|
||||
assert isolines_config.service_type is 'mapbox_isolines'
|
||||
else:
|
||||
assert isolines_config.service_type is 'here_isolines'
|
||||
assert isolines_config.isolines_quota == 100
|
||||
@ -215,9 +223,10 @@ class TestIsolinesUserConfig(TestCase):
|
||||
'test_user')
|
||||
assert isolines_config.soft_isolines_limit is False
|
||||
|
||||
|
||||
class TestIsolinesOrgConfig(TestCase):
|
||||
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
|
||||
ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
|
||||
@ -79,6 +79,7 @@ def increment_service_uses(redis_conn, username, orgname=None,
|
||||
def plpy_mock_config():
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
|
||||
|
||||
@ -0,0 +1,36 @@
|
||||
import unittest
|
||||
from mock import Mock
|
||||
from cartodb_services.mapbox import MapboxGeocoder
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
|
||||
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
|
||||
INVALID_TOKEN = 'invalid_token'
|
||||
VALID_ADDRESS = 'Calle Siempreviva 3, Valladolid'
|
||||
WELL_KNOWN_LONGITUDE = -4.730947
|
||||
WELL_KNOWN_LATITUDE = 41.668654
|
||||
|
||||
|
||||
class MapboxGeocoderTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.geocoder = MapboxGeocoder(token=VALID_TOKEN, logger=Mock())
|
||||
|
||||
def test_invalid_token(self):
|
||||
invalid_geocoder = MapboxGeocoder(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_geocoder.geocode(VALID_ADDRESS)
|
||||
|
||||
def test_valid_request(self):
|
||||
place = self.geocoder.geocode(VALID_ADDRESS)
|
||||
|
||||
self.assertEqual(place[0], WELL_KNOWN_LONGITUDE)
|
||||
self.assertEqual(place[1], WELL_KNOWN_LATITUDE)
|
||||
|
||||
def test_valid_request_namedplace(self):
|
||||
place = self.geocoder.geocode(searchtext='Barcelona')
|
||||
|
||||
assert place
|
||||
|
||||
def test_valid_request_namedplace2(self):
|
||||
place = self.geocoder.geocode(searchtext='New York', country='us')
|
||||
|
||||
assert place
|
||||
@ -0,0 +1,37 @@
|
||||
import unittest
|
||||
from mock import Mock
|
||||
from cartodb_services.mapbox.isolines import MapboxIsolines
|
||||
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
|
||||
from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
|
||||
from cartodb_services.mapbox.routing import MapboxRouting
|
||||
from cartodb_services.tools import Coordinate
|
||||
from cartodb_services.tools.coordinates import (validate_coordinates,
|
||||
marshall_coordinates)
|
||||
|
||||
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
|
||||
VALID_ORIGIN = Coordinate(-73.989, 40.733)
|
||||
|
||||
|
||||
class MapboxIsolinesTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
matrix_client = MapboxMatrixClient(token=VALID_TOKEN, logger=Mock())
|
||||
self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock())
|
||||
|
||||
def test_calculate_isochrone(self):
|
||||
time_ranges = [300, 900]
|
||||
solution = self.mapbox_isolines.calculate_isochrone(
|
||||
origin=VALID_ORIGIN,
|
||||
profile=DEFAULT_PROFILE,
|
||||
time_ranges=time_ranges)
|
||||
|
||||
assert solution
|
||||
|
||||
def test_calculate_isodistance(self):
|
||||
distance_range = 10000
|
||||
solution = self.mapbox_isolines.calculate_isodistance(
|
||||
origin=VALID_ORIGIN,
|
||||
profile=DEFAULT_PROFILE,
|
||||
distance_range=distance_range)
|
||||
|
||||
assert solution
|
||||
57
server/lib/python/cartodb_services/test/test_mapboxmatrix.py
Normal file
57
server/lib/python/cartodb_services/test/test_mapboxmatrix.py
Normal file
@ -0,0 +1,57 @@
|
||||
import unittest
|
||||
from mock import Mock
|
||||
from cartodb_services.mapbox import MapboxMatrixClient
|
||||
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.tools import Coordinate
|
||||
|
||||
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
|
||||
INVALID_TOKEN = 'invalid_token'
|
||||
VALID_ORIGIN = Coordinate(-73.989, 40.733)
|
||||
VALID_TARGET = Coordinate(-74, 40.733)
|
||||
VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET]
|
||||
NUM_COORDINATES_MAX = 25
|
||||
INVALID_COORDINATES_EMPTY = []
|
||||
INVALID_COORDINATES_MIN = [VALID_ORIGIN]
|
||||
INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \
|
||||
[VALID_TARGET
|
||||
for x in range(0, NUM_COORDINATES_MAX + 1)]
|
||||
VALID_PROFILE = DEFAULT_PROFILE
|
||||
INVALID_PROFILE = 'invalid_profile'
|
||||
|
||||
|
||||
class MapboxMatrixTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.matrix_client = MapboxMatrixClient(token=VALID_TOKEN,
|
||||
logger=Mock())
|
||||
|
||||
def test_invalid_profile(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.matrix_client.matrix(VALID_COORDINATES,
|
||||
INVALID_PROFILE)
|
||||
|
||||
def test_invalid_coordinates_empty(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.matrix_client.matrix(INVALID_COORDINATES_EMPTY,
|
||||
VALID_PROFILE)
|
||||
|
||||
def test_invalid_coordinates_max(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.matrix_client.matrix(INVALID_COORDINATES_MAX,
|
||||
VALID_PROFILE)
|
||||
|
||||
def test_invalid_coordinates_min(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.matrix_client.matrix(INVALID_COORDINATES_MIN,
|
||||
VALID_PROFILE)
|
||||
|
||||
def test_invalid_token(self):
|
||||
invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_matrix.matrix(VALID_COORDINATES,
|
||||
VALID_PROFILE)
|
||||
|
||||
def test_valid_request(self):
|
||||
distance_matrix = self.matrix_client.matrix(VALID_COORDINATES,
|
||||
VALID_PROFILE)
|
||||
assert distance_matrix
|
||||
@ -0,0 +1,63 @@
|
||||
import unittest
|
||||
from mock import Mock
|
||||
from cartodb_services.mapbox import MapboxRouting
|
||||
from cartodb_services.mapbox.routing import DEFAULT_PROFILE
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from cartodb_services.tools import Coordinate
|
||||
|
||||
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
|
||||
INVALID_TOKEN = 'invalid_token'
|
||||
VALID_WAYPOINTS = [Coordinate(-73.989, 40.733), Coordinate(-74, 40.733)]
|
||||
NUM_WAYPOINTS_MAX = 25
|
||||
INVALID_WAYPOINTS_EMPTY = []
|
||||
INVALID_WAYPOINTS_MIN = [Coordinate(-73.989, 40.733)]
|
||||
INVALID_WAYPOINTS_MAX = [Coordinate(-73.989, 40.733)
|
||||
for x in range(0, NUM_WAYPOINTS_MAX + 2)]
|
||||
VALID_PROFILE = DEFAULT_PROFILE
|
||||
INVALID_PROFILE = 'invalid_profile'
|
||||
|
||||
WELL_KNOWN_SHAPE = [(40.73312, -73.98891), (40.73353, -73.98987),
|
||||
(40.73398, -73.99095), (40.73453, -73.99227),
|
||||
(40.73531, -73.99412), (40.73467, -73.99459),
|
||||
(40.73442, -73.99477), (40.73435, -73.99482),
|
||||
(40.73403, -73.99505), (40.73344, -73.99549),
|
||||
(40.73286, -73.9959), (40.73226, -73.99635),
|
||||
(40.73186, -73.99664), (40.73147, -73.99693),
|
||||
(40.73141, -73.99698), (40.73147, -73.99707),
|
||||
(40.73219, -73.99856), (40.73222, -73.99861),
|
||||
(40.73293, -74.00007), (40.733, -74.00001)]
|
||||
WELL_KNOWN_LENGTH = 1317.9
|
||||
|
||||
|
||||
class MapboxRoutingTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.routing = MapboxRouting(token=VALID_TOKEN, logger=Mock())
|
||||
|
||||
def test_invalid_profile(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.routing.directions(VALID_WAYPOINTS, INVALID_PROFILE)
|
||||
|
||||
def test_invalid_waypoints_empty(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.routing.directions(INVALID_WAYPOINTS_EMPTY, VALID_PROFILE)
|
||||
|
||||
def test_invalid_waypoints_min(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.routing.directions(INVALID_WAYPOINTS_MIN, VALID_PROFILE)
|
||||
|
||||
def test_invalid_waypoints_max(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.routing.directions(INVALID_WAYPOINTS_MAX, VALID_PROFILE)
|
||||
|
||||
def test_invalid_token(self):
|
||||
invalid_routing = MapboxRouting(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_routing.directions(VALID_WAYPOINTS,
|
||||
VALID_PROFILE)
|
||||
|
||||
def test_valid_request(self):
|
||||
route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE)
|
||||
|
||||
self.assertEqual(route.shape, WELL_KNOWN_SHAPE)
|
||||
self.assertEqual(route.length, WELL_KNOWN_LENGTH)
|
||||
assert route.duration # The duration may change between executions
|
||||
@ -6,7 +6,8 @@ import requests_mock
|
||||
from mock import Mock
|
||||
|
||||
from cartodb_services.mapzen import MapzenGeocoder
|
||||
from cartodb_services.mapzen.exceptions import MalformedResult, TimeoutException
|
||||
from cartodb_services.tools.exceptions import (MalformedResult,
|
||||
TimeoutException)
|
||||
|
||||
requests_mock.Mocker.TEST_PREFIX = 'test_'
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ import requests_mock
|
||||
from mock import Mock
|
||||
|
||||
from cartodb_services.mapzen import MapzenIsochrones
|
||||
from cartodb_services.mapzen.exceptions import ServiceException
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
|
||||
requests_mock.Mocker.TEST_PREFIX = 'test_'
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ from urlparse import urlparse, parse_qs
|
||||
from mock import Mock
|
||||
|
||||
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
|
||||
from cartodb_services.mapzen.exceptions import WrongParams
|
||||
from cartodb_services.tools.exceptions import WrongParams
|
||||
from cartodb_services.tools import Coordinate
|
||||
|
||||
requests_mock.Mocker.TEST_PREFIX = 'test_'
|
||||
|
||||
@ -35,5 +35,3 @@ class TestPolyline(TestCase):
|
||||
decoded_polyline_3 = self.polyline.decode('ax_~Jv`d~\wrcMa`qj}@dfqaBngtcAhb~Zncc}y@')
|
||||
|
||||
assert decoded_polyline_3 == original_polyline_3
|
||||
|
||||
|
||||
|
||||
@ -2,13 +2,15 @@ import requests
|
||||
from unittest import TestCase
|
||||
from nose.tools import assert_raises
|
||||
from datetime import datetime, date
|
||||
from cartodb_services.mapzen.qps import qps_retry
|
||||
from cartodb_services.mapzen.exceptions import ServiceException, TimeoutException
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
from cartodb_services.tools.exceptions import (ServiceException,
|
||||
TimeoutException)
|
||||
import requests_mock
|
||||
import mock
|
||||
|
||||
requests_mock.Mocker.TEST_PREFIX = 'test_'
|
||||
|
||||
|
||||
@requests_mock.Mocker()
|
||||
class TestQPS(TestCase):
|
||||
QPS_ERROR_MESSAGE = "Queries per second exceeded: Queries exceeded (10 allowed)"
|
||||
@ -26,7 +28,7 @@ class TestQPS(TestCase):
|
||||
return self.QPS_ERROR_MESSAGE
|
||||
|
||||
req_mock.register_uri('GET', 'http://localhost/test_qps',
|
||||
text=_text_cb)
|
||||
text=_text_cb)
|
||||
with self.assertRaises(TimeoutException):
|
||||
c = TestClass()
|
||||
c.test()
|
||||
|
||||
@ -97,29 +97,81 @@ class TestQuotaService(TestCase):
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user')
|
||||
def test_should_check_user_mapbox_geocoder_quota_correctly(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
provider='mapbox')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user', orgname='testorg')
|
||||
def test_should_check_org_mapbox_geocoder_quota_correctly(self):
|
||||
qs = self.__build_geocoder_quota_service('test_user',
|
||||
orgname='testorg',
|
||||
provider='mapbox')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user')
|
||||
def test_should_check_user_mapzen_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user', provider='mapzen')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_mapzen_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user', provider='mapzen',
|
||||
orgname='testorg')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_mapbox_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user', provider='mapbox')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_mapbox_routing_quota_correctly(self):
|
||||
qs = self.__build_routing_quota_service('test_user', provider='mapbox',
|
||||
orgname='testorg')
|
||||
qs.increment_success_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_success_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_mapzen_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user',
|
||||
provider='mapzen')
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_isolines_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_isolines_quota_correctly(self):
|
||||
def test_should_check_org_mapzen_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user',
|
||||
provider='mapzen',
|
||||
orgname='testorg')
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_isolines_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_user_mapbox_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user',
|
||||
provider='mapbox')
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
qs.increment_isolines_service_use(amount=1500000)
|
||||
assert qs.check_user_quota() is False
|
||||
|
||||
def test_should_check_org_mapbox_isolines_quota_correctly(self):
|
||||
qs = self.__build_isolines_quota_service('test_user',
|
||||
provider='mapbox',
|
||||
orgname='testorg')
|
||||
qs.increment_isolines_service_use()
|
||||
assert qs.check_user_quota() is True
|
||||
@ -179,7 +231,7 @@ class TestQuotaService(TestCase):
|
||||
username, orgname)
|
||||
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_routing_quota_service(self, username, provider='mapzen',
|
||||
def __build_routing_quota_service(self, username, provider,
|
||||
orgname=None, soft_limit=False,
|
||||
quota=100, end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, 'routing', quota, provider,
|
||||
@ -188,7 +240,7 @@ class TestQuotaService(TestCase):
|
||||
username, orgname)
|
||||
return QuotaService(routing_config, redis_connection=self.redis_conn)
|
||||
|
||||
def __build_isolines_quota_service(self, username, provider='mapzen',
|
||||
def __build_isolines_quota_service(self, username, provider,
|
||||
orgname=None, soft_limit=False,
|
||||
quota=100, end_date=datetime.today()):
|
||||
self.__prepare_quota_service(username, 'isolines', quota, provider,
|
||||
|
||||
@ -97,7 +97,6 @@ class TestUserService(TestCase):
|
||||
super(MockRedisWithCounter, self).__init__()
|
||||
self._zscore_counter = 0
|
||||
def zscore(self, *args):
|
||||
print args
|
||||
self._zscore_counter += 1
|
||||
return super(MockRedisWithCounter, self).zscore(*args)
|
||||
def zscore_counter(self):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user