From 86e5f6d317e711cd330c4cc515d12ee1d9428700 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 24 May 2016 18:18:58 +0200 Subject: [PATCH 1/5] Limit the maximum overview level to 23 Fixes #257 23 = 32 bits per integer - 1 sign bit - 8 bits px/tile --- scripts-available/CDB_Overviews.sql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index d1322a0..10b3f57 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -1,3 +1,11 @@ +CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel() +RETURNS INTEGER +AS $$ + BEGIN + RETURN 23; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + -- Information about tables in a schema. -- If the schema name parameter is NULL, then tables from all schemas -- that may contain user tables are returned. @@ -322,7 +330,7 @@ AS $$ WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1) ) FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy) - WHERE t.e > %2$s AND t.z < (base.z + %3$s) + WHERE t.e > %2$s AND t.z < (base.z + %3$s) AND t.z <= _CDB_MaxOverviewLevel() ) SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0; ', reloid::text, min_features, nz, n, c, reloid::oid) @@ -363,7 +371,7 @@ AS $$ -- find minimum z so that fd*ta(z) <= lim -- compute a rough 'feature density' value SELECT CDB_XYZ_Resolution(-8) INTO c; - RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2); + RETURN least(_CDB_MaxOverviewLevel(), ceil(log(2.0, (c*c*fd/lim)::numeric)/2)); END; $$ LANGUAGE PLPGSQL STABLE; From c3fada29a84f917ed18e3940b9032f534e979e9e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 25 May 2016 11:07:06 +0200 Subject: [PATCH 2/5] Comment how the max overview level is defined --- scripts-available/CDB_Overviews.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index 10b3f57..f91424a 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -1,7 +1,16 @@ +-- Maximum zoom level for which overviews may be created CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel() RETURNS INTEGER AS $$ BEGIN + -- Zoom level will be limited so that both tile coordinates + -- and gridding coordinates within a tile up to 1px + -- (i.e. tile coordinates / 256) + -- can be stored in a 32-bit signed integer. + -- We have 31 bits por positive numbers + -- For zoom level Z coordinates range from 0 to 2^Z-1, so they + -- need Z bits, and need 8 bits more to address pixels within a tile + -- (gridding), so we'll limit Z to a maximum of 31 - 8 RETURN 23; END; $$ LANGUAGE PLPGSQL IMMUTABLE; From 803b3671d046ba92b9666fd315d9e9ad9cb4d36d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 25 May 2016 11:21:12 +0200 Subject: [PATCH 3/5] Fixes for feature density computation * The initial iteration of the recursive feature density query shouldn't start beyond the maximum level * Correct off-by-one limit the rest of iterations --- scripts-available/CDB_Overviews.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index f91424a..d7e93d5 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -326,7 +326,7 @@ AS $$ FROM ext, base ), seed AS ( - SELECT xt, yt, base.z, ( + SELECT xt, yt, least(base.z, _CDB_MaxOverviewLevel()), ( SELECT count(*) FROM %1$s WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z) ) e @@ -339,7 +339,7 @@ AS $$ WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1) ) FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy) - WHERE t.e > %2$s AND t.z < (base.z + %3$s) AND t.z <= _CDB_MaxOverviewLevel() + WHERE t.e > %2$s AND t.z < (base.z + %3$s) AND t.z < _CDB_MaxOverviewLevel() ) SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0; ', reloid::text, min_features, nz, n, c, reloid::oid) From 3399f2b9a5bae7f7218fc7e635afdd097b74353d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 25 May 2016 14:00:55 +0200 Subject: [PATCH 4/5] Fix max overviews level usage Note that _CDB_Feature_Density_Ref_Z_Strategy returns the first level for which overviews should not be used, and that in some cases _CDB_Feature_Density should look beyond the max level to compute a feature density estimate. --- scripts-available/CDB_Overviews.sql | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index d7e93d5..b9ce97d 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -15,6 +15,15 @@ AS $$ END; $$ LANGUAGE PLPGSQL IMMUTABLE; +-- Maximum zoom level usable with integer coordinates +CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel() +RETURNS INTEGER +AS $$ + BEGIN + RETURN 31; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + -- Information about tables in a schema. -- If the schema name parameter is NULL, then tables from all schemas -- that may contain user tables are returned. @@ -314,7 +323,11 @@ AS $$ WITH RECURSIVE t(x, y, z, e) AS ( WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g), base AS ( - SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z + SELECT + least( + -floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)), + _CDB_MaxOverviewLevel()+1 + )::integer z FROM ext ), lim AS ( @@ -326,7 +339,7 @@ AS $$ FROM ext, base ), seed AS ( - SELECT xt, yt, least(base.z, _CDB_MaxOverviewLevel()), ( + SELECT xt, yt, base.z, ( SELECT count(*) FROM %1$s WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z) ) e @@ -339,7 +352,7 @@ AS $$ WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1) ) FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy) - WHERE t.e > %2$s AND t.z < (base.z + %3$s) AND t.z < _CDB_MaxOverviewLevel() + WHERE t.e > %2$s AND t.z < least(base.z + %3$s, _CDB_MaxZoomLevel()) ) SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0; ', reloid::text, min_features, nz, n, c, reloid::oid) @@ -380,7 +393,7 @@ AS $$ -- find minimum z so that fd*ta(z) <= lim -- compute a rough 'feature density' value SELECT CDB_XYZ_Resolution(-8) INTO c; - RETURN least(_CDB_MaxOverviewLevel(), ceil(log(2.0, (c*c*fd/lim)::numeric)/2)); + RETURN least(_CDB_MaxOverviewLevel()+1, ceil(log(2.0, (c*c*fd/lim)::numeric)/2)); END; $$ LANGUAGE PLPGSQL STABLE; From 9526f0448f8bcedcd114e64f3d2760e01e34f905 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 26 May 2016 18:25:11 +0200 Subject: [PATCH 5/5] Fix bug in feature density recursive query If the table had x and/or y columns they were picked by an inner select instead of the recursive arguments. Fixes #256 --- scripts-available/CDB_Overviews.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index b9ce97d..c1520f2 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -349,7 +349,7 @@ AS $$ UNION ALL SELECT x*2 + xx, y*2 + yy, t.z+1, ( SELECT count(*) FROM %1$s - WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1) + WHERE the_geom_webmercator && CDB_XYZ_Extent(t.x*2 + c.xx, t.y*2 + c.yy, t.z+1) ) FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy) WHERE t.e > %2$s AND t.z < least(base.z + %3$s, _CDB_MaxZoomLevel())