cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Quota and batch size checks fixes and tests

Browse Source
This commit is contained in:
Juan Ignacio Sánchez Lara 2018-06-28 13:06:52 +02:00
parent 379257b4b4
commit 45b8fc4ecf
4 changed files with 60 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
client/cdb_dataservices_client--0.25.0.sql
View File

@ -1992,6 +1992,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE
query_row_count integer;
enough_quota boolean;
remaining_quota integer;
cartodb_id_batch integer;
batches_n integer;
@ -2002,7 +2003,7 @@ DECLARE
temp_table_name text;
BEGIN
IF batch_size IS NULL THEN
batch_size := DEFAULT_BATCH_SIZE;
RAISE EXCEPTION 'batch_size can''t be null';
ELSIF batch_size > MAX_BATCH_SIZE THEN
RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1;
END IF;
@ -2012,7 +2013,11 @@ BEGIN
RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
query_row_count, query, country_column, state_column, city_column, street_column;
SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
IF enough_quota IS NOT NULL AND enough_quota THEN
IF enough_quota IS NOT NULL AND NOT enough_quota THEN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota
INTO remaining_quota
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
WHERE service = 'hires_geocoder';
RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
END IF;

7
client/sql/21_bulk_geocoding_functions.sql
View File

@ -4,6 +4,7 @@ RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE
query_row_count integer;
enough_quota boolean;
remaining_quota integer;
cartodb_id_batch integer;
batches_n integer;
@ -14,7 +15,7 @@ DECLARE
temp_table_name text;
BEGIN
IF batch_size IS NULL THEN
batch_size := DEFAULT_BATCH_SIZE;
RAISE EXCEPTION 'batch_size can''t be null';
ELSIF batch_size > MAX_BATCH_SIZE THEN
RAISE EXCEPTION 'batch_size must be lower than %', MAX_BATCH_SIZE + 1;
END IF;
@ -25,6 +26,10 @@ BEGIN
query_row_count, query, country_column, state_column, city_column, street_column;
SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
IF enough_quota IS NOT NULL AND NOT enough_quota THEN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota
INTO remaining_quota
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
WHERE service = 'hires_geocoder';
RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
END IF;

21
client/test/expected/21_bulk_geocoding_functions_test.out Normal file
View File

@ -0,0 +1,21 @@
\set VERBOSITY terse
-- Test bulk size mandatory
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
ERROR: batch_size can't be null
-- Test quota check by mocking quota 0
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN as $$
SELECT FALSE;
$$ LANGUAGE SQL;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info ()
RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$
SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider;
$$ LANGUAGE SQL;
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
ERROR: Remaining quota: 0. Estimated cost: 1
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;

26
client/test/sql/21_bulk_geocoding_functions_test.sql Normal file
View File

@ -0,0 +1,26 @@
\set VERBOSITY terse
-- Test bulk size mandatory
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
-- Test quota check by mocking quota 0
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN as $$
SELECT FALSE;
$$ LANGUAGE SQL;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info ()
RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$
SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider;
$$ LANGUAGE SQL;
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;