cdb_bulk_geocode_street_point functions

client/cdb_dataservices_client--0.24.0.sql

@@ -63,6 +63,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
     the_geom geometry(Multipolygon,4326)
 );
 
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
 CREATE TYPE cdb_dataservices_client.simple_route AS (
     shape geometry(LineString,4326),
     length real,
@@ -400,6 +406,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
 -- These are the only ones with permissions to publicuser role
 -- and should also be the only ones with SECURITY DEFINER
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (searchtext jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
 RETURNS Geometry AS $$
 DECLARE
@@ -2377,6 +2408,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
 -- Exception-safe private DataServices API function
 --
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe (searchtext jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_bulk_geocode_street_point(username, orgname, searchtext);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+--
+-- Exception-safe private DataServices API function
+--
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
 RETURNS Geometry AS $$
 DECLARE
@@ -4301,6 +4368,14 @@ RETURNS Geometry AS $$
   
   SELECT cdb_dataservices_server.cdb_geocode_street_point (username, orgname, searchtext, city, state_province, country);
   
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (username text, orgname text, searchtext jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_bulk_geocode_street_point (username, orgname, searchtext);
+  
 $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
 DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
 CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
@@ -4809,6 +4884,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point_e
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
 
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(searchtext jsonb) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_exception_safe(searchtext jsonb )  TO publicuser;
+
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
 

client/renderer/interface.yaml

@@ -70,6 +70,13 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: cdb_bulk_geocode_street_point
+  return_type: SETOF cdb_dataservices_client.geocoding
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: searchtext,  type: jsonb }
+
 - name: cdb_here_geocode_street_point
   return_type: Geometry
   params:

client/sql/16_custom_types.sql

@@ -4,6 +4,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
     the_geom geometry(Multipolygon,4326)
 );
 
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
 CREATE TYPE cdb_dataservices_client.simple_route AS (
     shape geometry(LineString,4326),
     length real,

server/extension/cdb_dataservices_server--0.31.0.sql

@@ -2341,6 +2341,13 @@ RETURNS VOID AS $$
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_server_rate_limits(config)
 $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
+CREATE TYPE cdb_dataservices_server.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
 RETURNS Geometry AS $$
   from cartodb_services.metrics import QuotaService

server/extension/sql/21_bulk_geocode_street.sql

@@ -0,0 +1,75 @@
+-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
+CREATE TYPE cdb_dataservices_server.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'searchtext': searchtext}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_bulk_google_geocode_street_point($1, $2, $3); ", ["text", "text", "jsonb"])
+      result = plpy.execute(google_plan, [username, orgname, searchtext])
+      return result
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searchtext jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.tools import LegacyServiceManager,QuotaExceededException,Logger
+  from cartodb_services.google import GoogleMapsGeocoder
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+
+  logger = Logger(logger_config)
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits(quota=False)
+    geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
+    geocode_results = geocoder.bulk_geocode(searchtext=searchtext)
+    if geocode_results:
+      results = []
+      for result in geocode_results:
+        if result[1]:
+          plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"])
+          point = plpy.execute(plan, result[1], 1)[0]
+          results.append([result[0], point['the_geom'], None])
+        else:
+          results.append([result[0], None, None])
+      service_manager.quota_service.increment_success_service_use(len(results))
+      return results
+    else:
+      service_manager.quota_service.increment_empty_service_use(len(searchtext))
+      return []
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use(len(searchtext))
+    return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to bulk geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to bulk geocode street point using google maps')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+