From b0c1948c14c015c607555ce586b1f6301d7679ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 18 Jul 2018 14:29:50 +0200 Subject: [PATCH] TL;DR: safer deployment and minor fixes - Instead of modifying cdb_service_quota_info to return max_batch_size, a new type (service_quota_info_batch) and a new function (cdb_service_quota_info_batch) are created. That makes deployment safe. - Fixes geocoding with forced batch size 1. - Improves namespacing for count_estimate (-> cdb_dataservices_client.cdb_count_estimate). - Improves namespacing for jsonb_array_casttext (-> cdb_dataservices_client.cdb_jsonb_array_casttext). --- ...db_dataservices_client--0.24.0--0.25.0.sql | 106 ++++++++++++++++-- ...db_dataservices_client--0.25.0--0.24.0.sql | 18 +-- client/cdb_dataservices_client--0.25.0.sql | 93 +++++++++++++-- client/renderer/interface.yaml | 7 ++ client/sql/05_utils.sql | 4 +- client/sql/16_custom_types.sql | 8 ++ client/sql/21_bulk_geocoding_functions.sql | 9 +- .../21_bulk_geocoding_functions_test.out | 10 +- .../sql/21_bulk_geocoding_functions_test.sql | 10 +- ...db_dataservices_server--0.31.0--0.32.0.sql | 94 ++++++---------- ...db_dataservices_server--0.32.0--0.31.0.sql | 67 +---------- .../cdb_dataservices_server--0.32.0.sql | 62 ++++++++-- server/extension/sql/200_quotas.sql | 59 ++++++++-- .../extension/sql/21_bulk_geocode_street.sql | 3 +- .../cartodb_services/tools/log.py | 4 + test/integration/test_street_functions.py | 18 +++ 16 files changed, 375 insertions(+), 197 deletions(-) diff --git a/client/cdb_dataservices_client--0.24.0--0.25.0.sql b/client/cdb_dataservices_client--0.24.0--0.25.0.sql index 2bebf96..fd742f5 100644 --- a/client/cdb_dataservices_client--0.24.0--0.25.0.sql +++ b/client/cdb_dataservices_client--0.24.0--0.25.0.sql @@ -7,7 +7,7 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -23,18 +23,30 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE;-- - CREATE TYPE cdb_dataservices_client.geocoding AS ( cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb ); -ALTER TYPE cdb_dataservices_client.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC +); + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ @@ -55,12 +67,33 @@ BEGIN RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + -- -- Public dataservices API function -- -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ @@ -74,13 +107,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -120,12 +152,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -133,9 +164,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; @@ -143,6 +171,10 @@ BEGIN END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE @@ -175,19 +207,69 @@ BEGIN END; END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + -- -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; + DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb); CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ CONNECT cdb_dataservices_client._server_conn_str(); - + SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches); + +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; + +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser; diff --git a/client/cdb_dataservices_client--0.25.0--0.24.0.sql b/client/cdb_dataservices_client--0.25.0--0.24.0.sql index 98816a6..15b4950 100644 --- a/client/cdb_dataservices_client--0.25.0--0.24.0.sql +++ b/client/cdb_dataservices_client--0.25.0--0.24.0.sql @@ -6,20 +6,24 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- HERE goes your code to upgrade/downgrade -DROP FUNCTION IF EXISTS count_estimate(query text); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_count_estimate(query text); -DROP FUNCTION IF EXISTS jsonb_array_casttext(jsonb); - -ALTER TYPE cdb_dataservices_client.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb); -DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, - text, text, text, text, integer); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_quota_info_batch(); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, text, text, text, text, integer); DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe (); + DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb); -DROP TYPE IF EXISTS cdb_dataservices_client.geocoding; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (text, text); +DROP TYPE IF EXISTS cdb_dataservices_client.service_quota_info_batch; + +DROP TYPE IF EXISTS cdb_dataservices_client.geocoding; diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index b67c219..5ef3955 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -5,7 +5,7 @@ -- Make sure we have a sane search path to create/update the extension SET search_path = "$user",cartodb,public,cdb_dataservices_client; -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -21,7 +21,7 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE;-- -- Geocoder server connection config @@ -113,6 +113,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( ); CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( service cdb_dataservices_client.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -1601,6 +1609,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN AS $$ DECLARE @@ -2005,13 +2038,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -2051,12 +2083,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -2064,9 +2095,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; @@ -4201,6 +4229,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; -- Exception-safe private DataServices API function -- +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe () +RETURNS SETOF service_quota_info_batch AS $$ +DECLARE + + username text; + orgname text; + _returned_sqlstate TEXT; + _message_text TEXT; + _pg_exception_context TEXT; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + + BEGIN + RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname); + EXCEPTION + WHEN OTHERS THEN + GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, + _message_text = MESSAGE_TEXT, + _pg_exception_context = PG_EXCEPTION_CONTEXT; + RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; + + END; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE; +-- +-- Exception-safe private DataServices API function +-- + CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC) RETURNS BOOLEAN AS $$ DECLARE @@ -4845,6 +4909,14 @@ RETURNS SETOF service_quota_info AS $$ SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname); +$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text); +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text) +RETURNS SETOF service_quota_info_batch AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname); + $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC); CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC) @@ -5126,6 +5198,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exc GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser; + GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 394a5fa..54805cd 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -517,6 +517,13 @@ params: - {} +- name: cdb_service_quota_info_batch + return_type: SETOF service_quota_info_batch + multi_row: true + multi_field: true + params: + - {} + - name: cdb_enough_quota return_type: BOOLEAN params: diff --git a/client/sql/05_utils.sql b/client/sql/05_utils.sql index 31c8445..c245422 100644 --- a/client/sql/05_utils.sql +++ b/client/sql/05_utils.sql @@ -1,5 +1,5 @@ -- Taken from https://wiki.postgresql.org/wiki/Count_estimate -CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS +CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS $func$ DECLARE rec record; @@ -15,6 +15,6 @@ END $func$ LANGUAGE plpgsql; -- Taken from https://stackoverflow.com/a/48013356/351721 -CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$ SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x); $f$ LANGUAGE sql IMMUTABLE; \ No newline at end of file diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 7458ffa..15dc782 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -35,6 +35,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( ); CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + +CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS ( service cdb_dataservices_client.service_type, monthly_quota NUMERIC, used_quota NUMERIC, diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index dfbe515..5af493b 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -11,13 +11,12 @@ DECLARE batches_n integer; DEFAULT_BATCH_SIZE CONSTANT numeric := 100; MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000; - current_row_count integer ; temp_table_name text; BEGIN SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size INTO remaining_quota, max_batch_size - FROM cdb_dataservices_client.cdb_service_quota_info() csqi + FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi WHERE service = 'hires_geocoder'; RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size; @@ -57,12 +56,11 @@ BEGIN IF batches_n > 0 THEN FOR cartodb_id_batch in 0..(batches_n - 1) LOOP - EXECUTE format( 'WITH geocoding_data as (' || ' SELECT ' || ' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' || - ' floor((row_number() over ())::float/$1) as batch' || + ' floor((row_number() over () - 1)::float/$1) as batch' || ' FROM (%s) _x' || ') ' || 'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' || @@ -70,9 +68,6 @@ BEGIN 'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name) USING batch_size, cartodb_id_batch; - GET DIAGNOSTICS current_row_count = ROW_COUNT; - RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count; - END LOOP; END IF; diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out index d2f268b..c846a1d 100644 --- a/client/test/expected/21_bulk_geocoding_functions_test.out +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -1,7 +1,7 @@ \set VERBOSITY terse -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$ SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; $$ LANGUAGE SQL; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; @@ -15,7 +15,7 @@ ERROR: Remaining quota: 0. Estimated cost: 1 -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); ERROR: Remaining quota: 0. Estimated cost: 1 -DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql index d17f470..985133a 100644 --- a/client/test/sql/21_bulk_geocoding_functions_test.sql +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -1,8 +1,8 @@ \set VERBOSITY terse -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; -CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () -RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch () +RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$ SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size; $$ LANGUAGE SQL; @@ -18,9 +18,9 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod -- Test quota check by mocking quota 0 SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); -DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch; DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; -ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch; diff --git a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql index 9e7687c..45faaac 100644 --- a/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.31.0--0.32.0.sql @@ -4,75 +4,48 @@ -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT, + max_batch_size NUMERIC + ); + END IF; +END $$; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( username TEXT, orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ - from cartodb_services.metrics.user import UserMetricsService - from datetime import date +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - today = date.today() ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 - #-- Isolines - service = 'isolines' - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] - - #-- Hires Geocoder - service = 'hires_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] - - #-- Routing - service = 'routing' - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_routing_config, redis_conn) - - monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config.service_type, today) - soft_limit = user_routing_config.soft_limit - provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] - - #-- Observatory - service = 'observatory' - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - user_service = UserMetricsService(user_obs_config, redis_conn) - - monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config.service_type, today) - soft_limit = user_obs_config.soft_limit - provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; @@ -114,8 +87,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql index 549daca..2609e9d 100644 --- a/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0--0.31.0.sql @@ -4,75 +4,12 @@ -- HERE goes your code to upgrade/downgrade -ALTER TYPE cdb_dataservices_server.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( - username TEXT, - orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ - from cartodb_services.metrics.user import UserMetricsService - from datetime import date - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - today = date.today() - ret = [] - - #-- Isolines - service = 'isolines' - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) - - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Hires Geocoder - service = 'hires_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Routing - service = 'routing' - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_routing_config = GD["user_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_routing_config, redis_conn) - - monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config.service_type, today) - soft_limit = user_routing_config.soft_limit - provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - #-- Observatory - service = 'observatory' - plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_config = GD["user_obs_config_{0}".format(username)] - user_service = UserMetricsService(user_obs_config, redis_conn) - - monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config.service_type, today) - soft_limit = user_obs_config.soft_limit - provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - - return ret -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_quota_info_batch(TEXT, TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_google_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(TEXT, TEXT, jsonb); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(TEXT, TEXT, jsonb); DROP TYPE IF EXISTS cdb_dataservices_server.geocoding; +DROP TYPE IF EXISTS cdb_dataservices_server.service_quota_info_batch; diff --git a/server/extension/cdb_dataservices_server--0.32.0.sql b/server/extension/cdb_dataservices_server--0.32.0.sql index dc82322..42d9b9e 100644 --- a/server/extension/cdb_dataservices_server--0.32.0.sql +++ b/server/extension/cdb_dataservices_server--0.32.0.sql @@ -1857,6 +1857,21 @@ BEGIN WHERE pg_type.typname = 'service_quota_info' AND pg_namespace.nspname = 'cdb_dataservices_server') THEN CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -1867,13 +1882,13 @@ BEGIN END IF; END $$; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( username TEXT, orgname TEXT) RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date - from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -1891,7 +1906,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Hires Geocoder service = 'hires_geocoder' @@ -1903,12 +1918,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Routing service = 'routing' @@ -1920,7 +1930,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Observatory service = 'observatory' @@ -1932,12 +1942,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, @@ -2385,8 +2424,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 6f34083..973d1ce 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -18,6 +18,21 @@ BEGIN WHERE pg_type.typname = 'service_quota_info' AND pg_namespace.nspname = 'cdb_dataservices_server') THEN CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE pg_type.typname = 'service_quota_info_batch' + AND pg_namespace.nspname = 'cdb_dataservices_server') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS ( service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -28,13 +43,13 @@ BEGIN END IF; END $$; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( username TEXT, orgname TEXT) RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date - from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -52,7 +67,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Hires Geocoder service = 'hires_geocoder' @@ -64,12 +79,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider - batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) - if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): - max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE - else: - max_batch_size = 1 - ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Routing service = 'routing' @@ -81,7 +91,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] #-- Observatory service = 'observatory' @@ -93,12 +103,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]] + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] return ret $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$ + from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + + ret = [] + for info in sqi: + if info['service'] == 'hires_geocoder': + provider = info['provider'] + batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None) + if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'): + max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE + else: + max_batch_size = 1 + + info['max_batch_size'] = max_batch_size + else: + info['max_batch_size'] = 1 + + ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]] + + return ret +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 309d323..cee0539 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -35,8 +35,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ raise Exception('Requested geocoder is not available') plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) - result = plpy.execute(plan, [username, orgname, searches]) - return result + return plpy.execute(plan, [username, orgname, searches]) $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index 500c8ab..f38febb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -94,7 +94,11 @@ class Logger: else: exception_message = '' + # Adding trace breaks tests + # trace = traceback.format_exc(15) + # message = '{}{}. Trace: {}'.format(text, exception_message, trace) message = '{}{}'.format(text, exception_message) + if self._check_plpy(): if level == 'debug': plpy.debug(message) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index ebcc26f..2d02bdb 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -230,6 +230,24 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): } self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + def test_batch_size_1(self): + query = "select *, st_x(the_geom), st_y(the_geom) " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \ + "'select * from jsonb_to_recordset(''[" \ + "{\"cartodb_id\": 1, \"address\": \"1900 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 2, \"address\": \"1901 amphitheatre parkway, mountain view, ca, us\"}," \ + "{\"cartodb_id\": 3, \"address\": \"1902 amphitheatre parkway, mountain view, ca, us\"}" \ + "]''::jsonb) as (cartodb_id integer, address text)', " \ + "'address', null, null, null, 1)" + response = self._run_authenticated(query) + + points_by_cartodb_id = { + 1: self.fixture_points['1900 amphitheatre parkway'], + 2: self.fixture_points['1901 amphitheatre parkway'], + 3: self.fixture_points['1902 amphitheatre parkway'], + } + self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id) + def test_city_column_geocoding(self): query = "select *, st_x(the_geom), st_y(the_geom) " \ "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point( " \