From 34fc6439d2f646d0cde6b603f051d5efcc8e607d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Mon, 11 Jun 2018 16:12:29 +0200 Subject: [PATCH] cdb_bulk_geocode_street_point functions --- client/cdb_dataservices_client--0.24.0.sql | 78 +++++++++++++++++++ client/renderer/interface.yaml | 7 ++ client/sql/16_custom_types.sql | 6 ++ .../cdb_dataservices_server--0.31.0.sql | 7 ++ .../extension/sql/21_bulk_geocode_street.sql | 75 ++++++++++++++++++ 5 files changed, 173 insertions(+) create mode 100644 server/extension/sql/21_bulk_geocode_street.sql diff --git a/client/cdb_dataservices_client--0.24.0.sql b/client/cdb_dataservices_client--0.24.0.sql index 14a3af6..48bfb15 100644 --- a/client/cdb_dataservices_client--0.24.0.sql +++ b/client/cdb_dataservices_client--0.24.0.sql @@ -63,6 +63,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( the_geom geometry(Multipolygon,4326) ); +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE TYPE cdb_dataservices_client.simple_route AS ( shape geometry(LineString,4326), length real, @@ -400,6 +406,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -2377,6 +2408,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL) RETURNS Geometry AS $$ DECLARE @@ -4301,6 +4368,14 @@ RETURNS Geometry AS $$ SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb) +RETURNS SETOF cdb_dataservices_client.geocoding AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text); CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) @@ -4809,6 +4884,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 874e99a..71a7358 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -70,6 +70,13 @@ - { name: state_province, type: text, default: 'NULL'} - { name: country, type: text, default: 'NULL'} +- name: cdb_bulk_geocode_street_point + return_type: SETOF cdb_dataservices_client.geocoding + multi_row: true + multi_field: true + params: + - { name: searchtext, type: jsonb } + - name: cdb_here_geocode_street_point return_type: Geometry params: diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 2b77cdb..1b74cac 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -4,6 +4,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS ( the_geom geometry(Multipolygon,4326) ); +CREATE TYPE cdb_dataservices_client.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE TYPE cdb_dataservices_client.simple_route AS ( shape geometry(LineString,4326), length real, diff --git a/server/extension/cdb_dataservices_server--0.31.0.sql b/server/extension/cdb_dataservices_server--0.31.0.sql index 2e7b27c..c4360c4 100644 --- a/server/extension/cdb_dataservices_server--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0.sql @@ -2341,6 +2341,13 @@ RETURNS VOID AS $$ config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) config_setter.set_server_rate_limits(config) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql new file mode 100644 index 0000000..994594e --- /dev/null +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -0,0 +1,75 @@ +-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset? +CREATE TYPE cdb_dataservices_server.geocoding AS ( + cartodb_id integer, + the_geom geometry(Multipolygon,4326), + metadata jsonb +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'searchtext': searchtext} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"]) + result = plpy.execute(google_plan, [username, orgname, searchtext]) + return result + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger + from cartodb_services.google import GoogleMapsGeocoder + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + + logger = Logger(logger_config) + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + + try: + service_manager.assert_within_limits(quota=False) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) + geocode_results = geocoder.bulk_geocode(searchtext=searchtext) + if geocode_results: + results = [] + for result in geocode_results: + if result[1]: + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) + point = plpy.execute(plan, result[1], 1)[0] + results.append([result[0], point['the_geom'], None]) + else: + results.append([result[0], None, None]) + service_manager.quota_service.increment_success_service_use(len(results)) + return results + else: + service_manager.quota_service.increment_empty_service_use(len(searchtext)) + return [] + except QuotaExceededException as qe: + service_manager.quota_service.increment_failed_service_use(len(searchtext)) + return [] + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to bulk geocode street point using google maps') + finally: + service_manager.quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +