cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cdb_bulk_geocode_street_point functions

Browse Source
This commit is contained in:
Juan Ignacio Sánchez Lara 2018-06-11 16:12:29 +02:00
parent 3f08d37ef7
commit 34fc6439d2
5 changed files with 173 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
client/cdb_dataservices_client--0.24.0.sql
View File

@ -63,6 +63,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
the_geom geometry(Multipolygon,4326)
);
CREATE TYPE cdb_dataservices_client.geocoding AS (
cartodb_id integer,
the_geom geometry(Multipolygon,4326),
metadata jsonb
);
CREATE TYPE cdb_dataservices_client.simple_route AS (
shape geometry(LineString,4326),
length real,
@ -400,6 +406,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
@ -2377,6 +2408,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE
username text;
orgname text;
_returned_sqlstate TEXT;
_message_text TEXT;
_pg_exception_context TEXT;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
BEGIN
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext);
EXCEPTION
WHEN OTHERS THEN
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
_message_text = MESSAGE_TEXT,
_pg_exception_context = PG_EXCEPTION_CONTEXT;
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
END;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
@ -4301,6 +4368,14 @@ RETURNS Geometry AS $$
SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
@ -4809,6 +4884,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser;

7
client/renderer/interface.yaml
View File

@ -70,6 +70,13 @@
- { name: state_province, type: text, default: 'NULL'}
- { name: country, type: text, default: 'NULL'}
- name: cdb_bulk_geocode_street_point
return_type: SETOF cdb_dataservices_client.geocoding
multi_row: true
multi_field: true
params:
- { name: searchtext, type: jsonb }
- name: cdb_here_geocode_street_point
return_type: Geometry
params:

6
client/sql/16_custom_types.sql
View File

@ -4,6 +4,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
the_geom geometry(Multipolygon,4326)
);
CREATE TYPE cdb_dataservices_client.geocoding AS (
cartodb_id integer,
the_geom geometry(Multipolygon,4326),
metadata jsonb
);
CREATE TYPE cdb_dataservices_client.simple_route AS (
shape geometry(LineString,4326),
length real,

7
server/extension/cdb_dataservices_server--0.31.0.sql
View File

@ -2341,6 +2341,13 @@ RETURNS VOID AS $$
config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
config_setter.set_server_rate_limits(config)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
CREATE TYPE cdb_dataservices_server.geocoding AS (
cartodb_id integer,
the_geom geometry(Multipolygon,4326),
metadata jsonb
);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService

75
server/extension/sql/21_bulk_geocode_street.sql Normal file
View File

@ -0,0 +1,75 @@
-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
CREATE TYPE cdb_dataservices_server.geocoding AS (
cartodb_id integer,
the_geom geometry(Multipolygon,4326),
metadata jsonb
);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'searchtext': searchtext}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"])
result = plpy.execute(google_plan, [username, orgname, searchtext])
return result
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger
from cartodb_services.google import GoogleMapsGeocoder
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
try:
service_manager.assert_within_limits(quota=False)
geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
geocode_results = geocoder.bulk_geocode(searchtext=searchtext)
if geocode_results:
results = []
for result in geocode_results:
if result[1]:
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"])
point = plpy.execute(plan, result[1], 1)[0]
results.append([result[0], point['the_geom'], None])
else:
results.append([result[0], None, None])
service_manager.quota_service.increment_success_service_use(len(results))
return results
else:
service_manager.quota_service.increment_empty_service_use(len(searchtext))
return []
except QuotaExceededException as qe:
service_manager.quota_service.increment_failed_service_use(len(searchtext))
return []
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to bulk geocode street point using google maps')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;