From 7ea428913aff050ed221fa90f9ccbe0df26c531d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 14 Feb 2018 17:50:06 +0100 Subject: [PATCH 1/4] Add limit to the grid-generating functions This adds an optinal maxcells parameter to the rectangle and hexagonal grids, with a default value of 262144 (2^18). An error occurs when the grids would need more cells. --- scripts-available/CDB_Hexagon.sql | 17 +++++++++++++++-- scripts-available/CDB_RectangleGrid.sql | 16 ++++++++++++++-- test/CDB_HexagonTest.sql | 6 ++++++ test/CDB_HexagonTest_expect | 3 +++ test/CDB_RectangleTest.sql | 5 +++++ test/CDB_RectangleTest_expect | 3 +++ 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 test/CDB_RectangleTest.sql create mode 100644 test/CDB_RectangleTest_expect diff --git a/scripts-available/CDB_Hexagon.sql b/scripts-available/CDB_Hexagon.sql index 68430b1..bb2b82d 100644 --- a/scripts-available/CDB_Hexagon.sql +++ b/scripts-available/CDB_Hexagon.sql @@ -10,6 +10,10 @@ AS $$ WHERE path[1] % 2 != 0 $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE; + +-- In older versions of the extension, CDB_HexagonGrid had a different signature +DROP FUNCTION IF EXISTS cartodb.CDB_HexagonGrid(GEOMETRY, FLOAT8, GEOMETRY); + -- -- Fill given extent with an hexagonal coverage -- @@ -25,10 +29,13 @@ $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE; -- If omitted the origin will be 0,0. -- The parameter is checked for having the same SRID -- as the extent. --- -- +-- @param maxcells Optional maximum number of grid cells to generate; +-- if the grid requires more cells to cover the extent +-- and exception will occur. +---- -- DROP FUNCTION IF EXISTS CDB_HexagonGrid(ext GEOMETRY, side FLOAT8); -CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL) +CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512) RETURNS SETOF GEOMETRY AS $$ DECLARE @@ -107,6 +114,12 @@ BEGIN vstartary := ARRAY[ vstart - (vstep/2.0), vstart ]; END IF; + If maxcells IS NOT NULL AND maxcells > 0 THEN + IF CEIL((CEIL((vend-vstart)/(vstep/2.0)) * CEIL((hend-hstart)/(hstep*2.0/3.0)))/3.0)::integer > maxcells THEN + RAISE EXCEPTION 'The requested grid is too big to be rendered'; + END IF; + END IF; + vstartidx := abs(hskip)%2; RAISE DEBUG 'vstartary: % : %', vstartary[1], vstartary[2]; diff --git a/scripts-available/CDB_RectangleGrid.sql b/scripts-available/CDB_RectangleGrid.sql index c2bc166..6c01e03 100644 --- a/scripts-available/CDB_RectangleGrid.sql +++ b/scripts-available/CDB_RectangleGrid.sql @@ -1,3 +1,6 @@ +-- In older versions of the extension, CDB_RectangleGrid had a different signature +DROP FUNCTION IF EXISTS cartodb.CDB_RectangleGrid(GEOMETRY, FLOAT8, FLOAT8, GEOMETRY); + -- -- Fill given extent with a rectangular coverage -- @@ -14,9 +17,12 @@ -- If omitted the origin will be 0,0. -- The parameter is checked for having the same SRID -- as the extent. --- -- -CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL) +-- @param maxcells Optional maximum number of grid cells to generate; +-- if the grid requires more cells to cover the extent +-- and exception will occur. +-- +CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512) RETURNS SETOF GEOMETRY AS $$ DECLARE @@ -79,6 +85,12 @@ BEGIN --RAISE DEBUG 'hend: %', hend; --RAISE DEBUG 'vend: %', vend; + If maxcells IS NOT NULL AND maxcells > 0 THEN + IF ((hend - hstart)/hstep * (vend - vstart)/vstep)::integer > maxcells THEN + RAISE EXCEPTION 'The requested grid is too big to be rendered'; + END IF; + END IF; + x := hstart; WHILE x < hend LOOP -- over X y := vstart; diff --git a/test/CDB_HexagonTest.sql b/test/CDB_HexagonTest.sql index 66b52b1..156a916 100644 --- a/test/CDB_HexagonTest.sql +++ b/test/CDB_HexagonTest.sql @@ -45,3 +45,9 @@ WITH 0.002 as radius ), grid AS ( SELECT CDB_HexagonGrid(env, radius) AS cell from params) SELECT '#160', count(cell) > 23000 from grid; + +-- Check small grids are generated... +SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10); + +-- But large grids produce an error +SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1); diff --git a/test/CDB_HexagonTest_expect b/test/CDB_HexagonTest_expect index 08e74f2..a6b6133 100644 --- a/test/CDB_HexagonTest_expect +++ b/test/CDB_HexagonTest_expect @@ -1,2 +1,5 @@ 9|63|count / npoints #160|t +3886 +ERROR: The requested grid is too big to be rendered +CONTEXT: PL/pgSQL function cdb_hexagongrid(geometry,double precision,geometry,integer) line 80 at RAISE diff --git a/test/CDB_RectangleTest.sql b/test/CDB_RectangleTest.sql new file mode 100644 index 0000000..76ec66e --- /dev/null +++ b/test/CDB_RectangleTest.sql @@ -0,0 +1,5 @@ +-- Check small grids are generated... +SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10, 10); + +-- But large grids produce an error +SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1, 1); diff --git a/test/CDB_RectangleTest_expect b/test/CDB_RectangleTest_expect new file mode 100644 index 0000000..b2de4df --- /dev/null +++ b/test/CDB_RectangleTest_expect @@ -0,0 +1,3 @@ +10000 +ERROR: The requested grid is too big to be rendered +CONTEXT: PL/pgSQL function cdb_rectanglegrid(geometry,double precision,double precision,geometry,integer) line 64 at RAISE From cd4653ecc0347c2610ca525db7225f294d1ba5f4 Mon Sep 17 00:00:00 2001 From: Alberto Romeu Date: Thu, 15 Feb 2018 08:40:17 +0100 Subject: [PATCH 2/4] grant usage on cartodb_id sequence when sharing read write --- scripts-available/CDB_Organizations.sql | 1 + test/organization/test.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_Organizations.sql b/scripts-available/CDB_Organizations.sql index b49529d..9af6081 100644 --- a/scripts-available/CDB_Organizations.sql +++ b/scripts-available/CDB_Organizations.sql @@ -111,6 +111,7 @@ AS $$ BEGIN EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"'; EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"'; + EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE ' || pg_catalog.pg_get_serial_sequence(Format('%I.%I', from_schema, table_name), 'cartodb_id') || ' TO "' || to_role_name || '"'; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/organization/test.sh b/test/organization/test.sh index f32dfa7..b7b9d83 100644 --- a/test/organization/test.sh +++ b/test/organization/test.sh @@ -315,6 +315,8 @@ function test_member_1_cannot_grant_read_write_permission_to_other_schema_than_i } function test_member_2_can_write_to_member_1_table_after_write_permission_is_added() { + sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;" + sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')" sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10 @@ -322,6 +324,8 @@ function test_member_2_can_write_to_member_1_table_after_write_permission_is_add sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;' sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9 sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9 + + sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;" } function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table() { @@ -332,10 +336,14 @@ function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table } function test_member_1_removes_access_and_member_2_can_no_longer_write_to_the_table() { + sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;" + sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')" sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')" sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails + + sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;" } function test_giving_permissions_to_two_tables_and_removing_from_first_table_should_not_remove_from_second() { @@ -418,7 +426,7 @@ function test_cdb_usertables_should_work_with_orgusers() { sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)" sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);" sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser" - + sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)" From 0a0f2e294bcac924bac02e726aad99cfe1f0325c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 15 Feb 2018 10:27:19 +0100 Subject: [PATCH 3/4] Fix tests --- test/CDB_HexagonTest.sql | 3 +++ test/CDB_HexagonTest_expect | 2 +- test/CDB_RectangleTest.sql | 3 +++ test/CDB_RectangleTest_expect | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/CDB_HexagonTest.sql b/test/CDB_HexagonTest.sql index 156a916..038540d 100644 --- a/test/CDB_HexagonTest.sql +++ b/test/CDB_HexagonTest.sql @@ -1,3 +1,6 @@ +set client_min_messages to error; +\set VERBOSITY TERSE + -- Check correctness of an hexagons grid -- -- Cells must have no overlaps and have a number of diff --git a/test/CDB_HexagonTest_expect b/test/CDB_HexagonTest_expect index a6b6133..42c7fec 100644 --- a/test/CDB_HexagonTest_expect +++ b/test/CDB_HexagonTest_expect @@ -1,5 +1,5 @@ +SET 9|63|count / npoints #160|t 3886 ERROR: The requested grid is too big to be rendered -CONTEXT: PL/pgSQL function cdb_hexagongrid(geometry,double precision,geometry,integer) line 80 at RAISE diff --git a/test/CDB_RectangleTest.sql b/test/CDB_RectangleTest.sql index 76ec66e..a16b7fa 100644 --- a/test/CDB_RectangleTest.sql +++ b/test/CDB_RectangleTest.sql @@ -1,3 +1,6 @@ +set client_min_messages to error; +\set VERBOSITY TERSE + -- Check small grids are generated... SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10, 10); diff --git a/test/CDB_RectangleTest_expect b/test/CDB_RectangleTest_expect index b2de4df..cc1c961 100644 --- a/test/CDB_RectangleTest_expect +++ b/test/CDB_RectangleTest_expect @@ -1,3 +1,3 @@ +SET 10000 ERROR: The requested grid is too big to be rendered -CONTEXT: PL/pgSQL function cdb_rectanglegrid(geometry,double precision,double precision,geometry,integer) line 64 at RAISE From c1c55bf59f0814aacd8e114988da83a7f6ebe83f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 15 Feb 2018 17:22:42 +0100 Subject: [PATCH 4/4] Release 0.21.0 --- Makefile | 3 ++- NEWS.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 01e9325..2f31953 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.20.0 +EXTVERSION = 0.21.0 SED = sed AWK = awk @@ -84,6 +84,7 @@ UPGRADABLE = \ 0.19.1 \ 0.19.2 \ 0.20.0 \ + 0.21.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index 233454f..961e9d2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.21.0 (2018-02-15) +* Add optional parameter to limit the number of cells in grid-generation functions #322 +* Fix: grant usage on cartodb_id sequence when sharing read write #323 +* Fix: Change sed in-place for tmpfiles 524319 + 0.20.0 (2017-11-08) * Added VOLATILITY and PARALLEL categories to all functions