From 4d2abc7667bf62ba9c8963fce2609266691ae61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Fri, 22 Jun 2018 09:41:29 +0200 Subject: [PATCH] Simpler and more precise quota check --- client/cdb_dataservices_client--0.25.0.sql | 24 ++++++++-------------- client/sql/21_bulk_geocoding_functions.sql | 24 ++++++++-------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/client/cdb_dataservices_client--0.25.0.sql b/client/cdb_dataservices_client--0.25.0.sql index 088f949..6d87658 100644 --- a/client/cdb_dataservices_client--0.25.0.sql +++ b/client/cdb_dataservices_client--0.25.0.sql @@ -1989,29 +1989,21 @@ $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE - monthly_quota integer; - used_quota integer; - soft_limit boolean; - provider text; - - remaining_quota integer; - estimated_row_count integer; + query_row_count integer; + enough_quota boolean; cartodb_id_batch integer; batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; BEGIN - SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider - into monthly_quota, used_quota, soft_limit, provider - FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - remaining_quota := monthly_quota - used_quota; - estimated_row_count := count_estimate(query); - RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', - estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; - IF estimated_row_count > remaining_quota THEN - RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', + query_row_count, query, searchtext; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NOT NULL AND enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; diff --git a/client/sql/21_bulk_geocoding_functions.sql b/client/sql/21_bulk_geocoding_functions.sql index 12766fc..093cb60 100644 --- a/client/sql/21_bulk_geocoding_functions.sql +++ b/client/sql/21_bulk_geocoding_functions.sql @@ -1,29 +1,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) RETURNS SETOF cdb_dataservices_client.geocoding AS $$ DECLARE - monthly_quota integer; - used_quota integer; - soft_limit boolean; - provider text; - - remaining_quota integer; - estimated_row_count integer; + query_row_count integer; + enough_quota boolean; cartodb_id_batch integer; batches_n integer; BATCHES_SIZE CONSTANT numeric := 100; current_row_count integer ; BEGIN - SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider - into monthly_quota, used_quota, soft_limit, provider - FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder'; + EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count; - remaining_quota := monthly_quota - used_quota; - estimated_row_count := count_estimate(query); - RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', - estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; - IF estimated_row_count > remaining_quota THEN - RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count; + RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %', + query_row_count, query, searchtext; + SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota; + IF enough_quota IS NOT NULL AND enough_quota THEN + RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count; END IF; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n;