From 45b8fc4ecf6903892b4ff6a43bdf636e65735898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 28 Jun 2018 13:06:52 +0200 Subject: [PATCH] Quota and batch size checks fixes and tests --- client/cdb_dataservices_client--0.25.0.sql | 9 +++++-- client/sql/21_bulk_geocoding_functions.sql | 7 ++++- .../21_bulk_geocoding_functions_test.out | 21 +++++++++++++++ .../sql/21_bulk_geocoding_functions_test.sql | 26 +++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 client/test/expected/21_bulk_geocoding_functions_test.out create mode 100644 client/test/sql/21_bulk_geocoding_functions_test.sql diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index be72fe5..2e906d7 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1992,6 +1992,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; + remaining_quota integer; cartodb_id_batch integer; batches_n integer; @@ -2002,7 +2003,7 @@ DECLARE temp_table_name text; BEGIN IF batch_size IS NULL THEN - batch_size := DEFAULT_BATCH_SIZE; + RAISE EXCEPTION 'batch_size can''t be null'; ELSIF batch_size > MAX_BATCH_SIZE THEN RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; END IF; @@ -2012,7 +2013,11 @@ BEGIN RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %', query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; - IF enough_quota IS NOT NULL AND enough_quota THEN + IF enough_quota IS NOT NULL AND NOT enough_quota THEN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota + INTO remaining_quota + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 4fbd5d1..4314e7c 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -4,6 +4,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE query_row_count integer; enough_quota boolean; + remaining_quota integer; cartodb_id_batch integer; batches_n integer; @@ -14,7 +15,7 @@ DECLARE temp_table_name text; BEGIN IF batch_size IS NULL THEN - batch_size := DEFAULT_BATCH_SIZE; + RAISE EXCEPTION 'batch_size can''t be null'; ELSIF batch_size > MAX_BATCH_SIZE THEN RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1; END IF; @@ -25,6 +26,10 @@ BEGIN query_row_count, query, country_column, state_column, city_column, street_column; SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; IF enough_quota IS NOT NULL AND NOT enough_quota THEN + SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota + INTO remaining_quota + FROM cdb_dataservices_client.cdb_service_quota_info() csqi + WHERE service = 'hires_geocoder'; RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; diff --git a/client/test/expected/21_bulk_geocoding_functions_test.out b/client/test/expected/21_bulk_geocoding_functions_test.out new file mode 100644 index 0000000..7ca70db --- /dev/null +++ b/client/test/expected/21_bulk_geocoding_functions_test.out @@ -0,0 +1,21 @@ +\set VERBOSITY terse +-- Test bulk size mandatory +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); +ERROR: batch_size can't be null +-- Test quota check by mocking quota 0 +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT FALSE; +$$ LANGUAGE SQL; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; +$$ LANGUAGE SQL; +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); +ERROR: Remaining quota: 0. Estimated cost: 1 +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; diff --git a/client/test/sql/21_bulk_geocoding_functions_test.sql b/client/test/sql/21_bulk_geocoding_functions_test.sql new file mode 100644 index 0000000..5433bd7 --- /dev/null +++ b/client/test/sql/21_bulk_geocoding_functions_test.sql @@ -0,0 +1,26 @@ +\set VERBOSITY terse + +-- Test bulk size mandatory +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null); + +-- Test quota check by mocking quota 0 +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) +RETURNS BOOLEAN as $$ + SELECT FALSE; +$$ LANGUAGE SQL; + +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked; +CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info () +RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$ + SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider; +$$ LANGUAGE SQL; + +SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain'''); + +DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info; +DROP FUNCTION cdb_dataservices_client.cdb_enough_quota; + +ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info; +ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota; +