cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simpler and more precise quota check

Browse Source
This commit is contained in:
Juan Ignacio Sánchez Lara 2018-06-22 09:41:29 +02:00
parent 58d70e252f
commit 4d2abc7667
2 changed files with 16 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
client/cdb_dataservices_client--0.25.0.sql
View File

@ -1989,29 +1989,21 @@ $$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$ RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE DECLARE
monthly_quota integer; query_row_count integer;
used_quota integer; enough_quota boolean;
soft_limit boolean;
provider text;
remaining_quota integer;
estimated_row_count integer;
cartodb_id_batch integer; cartodb_id_batch integer;
batches_n integer; batches_n integer;
BATCHES_SIZE CONSTANT numeric := 100; BATCHES_SIZE CONSTANT numeric := 100;
current_row_count integer ; current_row_count integer ;
BEGIN BEGIN
SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count;
into monthly_quota, used_quota, soft_limit, provider
FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder';
remaining_quota := monthly_quota - used_quota; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %',
estimated_row_count := count_estimate(query); query_row_count, query, searchtext;
RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; IF enough_quota IS NOT NULL AND enough_quota THEN
IF estimated_row_count > remaining_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count;
END IF; END IF;
EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n;

24
client/sql/21_bulk_geocoding_functions.sql
View File

@ -1,29 +1,21 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text) CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text, searchtext text)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$ RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE DECLARE
monthly_quota integer; query_row_count integer;
used_quota integer; enough_quota boolean;
soft_limit boolean;
provider text;
remaining_quota integer;
estimated_row_count integer;
cartodb_id_batch integer; cartodb_id_batch integer;
batches_n integer; batches_n integer;
BATCHES_SIZE CONSTANT numeric := 100; BATCHES_SIZE CONSTANT numeric := 100;
current_row_count integer ; current_row_count integer ;
BEGIN BEGIN
SELECT csqi.monthly_quota, csqi.used_quota, csqi.soft_limit, csqi.provider EXECUTE format('SELECT COUNT(1) from (%s) _x', query) INTO query_row_count;
into monthly_quota, used_quota, soft_limit, provider
FROM cdb_dataservices_client.cdb_service_quota_info() csqi WHERE service = 'hires_geocoder';
remaining_quota := monthly_quota - used_quota; RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; searchtext: %',
estimated_row_count := count_estimate(query); query_row_count, query, searchtext;
RAISE DEBUG 'cdb_bulk_geocode_street_point --> estimated: %, remaining: %, monthly_quota: %, used_quota: %, soft_limit: %, provider: %', SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
estimated_row_count, remaining_quota, monthly_quota, used_quota, soft_limit, provider; IF enough_quota IS NOT NULL AND enough_quota THEN
IF estimated_row_count > remaining_quota THEN RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, estimated_row_count;
END IF; END IF;
EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n; EXECUTE format('SELECT ceil(max(cartodb_id)::float/%s) FROM (%s) _x', BATCHES_SIZE, query) INTO batches_n;