cartodb4/cartodb-postgresql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Exclude overviews from user data size

Browse Source 
Fixes #261

Some internal functions from the Overviews module
have been moved to a separate file because they're now
used from Quaota function.
This commit is contained in:
Javier Goizueta 2016-06-07 13:20:55 +02:00
parent 58fd5d4060
commit 1f01ecae30
7 changed files with 179 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
scripts-available/CDB_Overviews.sql
View File

@ -1,175 +1,3 @@
-- Maximum zoom level for which overviews may be created
CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel()
RETURNS INTEGER
AS $$
BEGIN
-- Zoom level will be limited so that both tile coordinates
-- and gridding coordinates within a tile up to 1px
-- (i.e. tile coordinates / 256)
-- can be stored in a 32-bit signed integer.
-- We have 31 bits por positive numbers
-- For zoom level Z coordinates range from 0 to 2^Z-1, so they
-- need Z bits, and need 8 bits more to address pixels within a tile
-- (gridding), so we'll limit Z to a maximum of 31 - 8
RETURN 23;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Maximum zoom level usable with integer coordinates
CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel()
RETURNS INTEGER
AS $$
BEGIN
RETURN 31;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Information about tables in a schema.
-- If the schema name parameter is NULL, then tables from all schemas
-- that may contain user tables are returned.
-- For each table, the regclass, schema name and table name are returned.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
AS $$
SELECT
c.oid::regclass AS table_regclass,
n.nspname::text AS schema_name,
c.relname::text AS table_relname
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
AND CASE WHEN schema_name IS NULL
THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
ELSE n.nspname = schema_name
END;
$$ LANGUAGE 'sql';
-- Pattern that can be used to detect overview tables and Extract
-- the intended zoom level from the table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
RETURNS TEXT
AS $$
BEGIN
RETURN '\A_vovw_(\d+)_';
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- substring(tablename from _CDB_OverviewTableDiscriminator())
-- Pattern matched by the overview tables of a given base table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
RETURNS TEXT
AS $$
BEGIN
RETURN _CDB_OverviewTableDiscriminator() || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-- Name of an overview table, given the base table name and the Z level
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
RETURNS TEXT
AS $$
BEGIN
RETURN '_vovw_' || z::text || '_' || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Condition to check if a tabla is an overview table of some base table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
RETURNS BOOLEAN
AS $$
BEGIN
RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Extract the Z level from an overview table name
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
RETURNS INTEGER
AS $$
BEGIN
RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Name of the base table corresponding to an overview table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
RETURNS TEXT
AS $$
BEGIN
IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
RETURN overview_table;
ELSE
RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
END IF;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
RETURNS REGCLASS
AS $$
DECLARE
table_name TEXT;
schema_name TEXT;
base_name TEXT;
base_table REGCLASS;
BEGIN
SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
base_name := _CDB_OverviewBaseTableName(table_name);
IF base_name != table_name THEN
base_table := Format('%I.%I', schema_name, base_name)::regclass;
ELSE
base_table := overview_table;
END IF;
RETURN base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Schema and relation names of a table given its reloid
-- Scope: private.
-- Parameters
-- reloid: oid of the table.
-- Return (schema_name, table_name)
-- note that returned names will be quoted if necessary
CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
AS $$
BEGIN
SELECT n.nspname, c.relname
INTO STRICT schema_name, table_name
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = reloid;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Schema and relation names of a table given its reloid
-- Scope: private.
-- Parameters
-- reloid: oid of the table.
-- Return (schema_name, table_name)
-- note that returned names will be quoted if necessary
CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
RETURNS TEXT
AS $$
DECLARE
schema_name TEXT;
BEGIN
SELECT n.nspname
INTO STRICT schema_name
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = reloid;
RETURN schema_name;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Remove a dataset's existing overview tables.
-- Scope: public
-- Parameters:

173
scripts-available/CDB_OverviewsSupport.sql Normal file
View File

@ -0,0 +1,173 @@
-- Auxiliary overviews FUNCTIONS
-- Maximum zoom level for which overviews may be created
CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel()
RETURNS INTEGER
AS $$
BEGIN
-- Zoom level will be limited so that both tile coordinates
-- and gridding coordinates within a tile up to 1px
-- (i.e. tile coordinates / 256)
-- can be stored in a 32-bit signed integer.
-- We have 31 bits por positive numbers
-- For zoom level Z coordinates range from 0 to 2^Z-1, so they
-- need Z bits, and need 8 bits more to address pixels within a tile
-- (gridding), so we'll limit Z to a maximum of 31 - 8
RETURN 23;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Maximum zoom level usable with integer coordinates
CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel()
RETURNS INTEGER
AS $$
BEGIN
RETURN 31;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Information about tables in a schema.
-- If the schema name parameter is NULL, then tables from all schemas
-- that may contain user tables are returned.
-- For each table, the regclass, schema name and table name are returned.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
AS $$
SELECT
c.oid::regclass AS table_regclass,
n.nspname::text AS schema_name,
c.relname::text AS table_relname
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
AND CASE WHEN schema_name IS NULL
THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
ELSE n.nspname = schema_name
END;
$$ LANGUAGE 'sql';
-- Pattern that can be used to detect overview tables and Extract
-- the intended zoom level from the table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
RETURNS TEXT
AS $$
BEGIN
RETURN '\A_vovw_(\d+)_';
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- substring(tablename from _CDB_OverviewTableDiscriminator())
-- Pattern matched by the overview tables of a given base table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
RETURNS TEXT
AS $$
BEGIN
RETURN _CDB_OverviewTableDiscriminator() || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-- Name of an overview table, given the base table name and the Z level
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
RETURNS TEXT
AS $$
BEGIN
RETURN '_vovw_' || z::text || '_' || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Condition to check if a tabla is an overview table of some base table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
RETURNS BOOLEAN
AS $$
BEGIN
RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Extract the Z level from an overview table name
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
RETURNS INTEGER
AS $$
BEGIN
RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Name of the base table corresponding to an overview table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
RETURNS TEXT
AS $$
BEGIN
IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
RETURN overview_table;
ELSE
RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
END IF;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
RETURNS REGCLASS
AS $$
DECLARE
table_name TEXT;
schema_name TEXT;
base_name TEXT;
base_table REGCLASS;
BEGIN
SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
base_name := _CDB_OverviewBaseTableName(table_name);
IF base_name != table_name THEN
base_table := Format('%I.%I', schema_name, base_name)::regclass;
ELSE
base_table := overview_table;
END IF;
RETURN base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Schema and relation names of a table given its reloid
-- Scope: private.
-- Parameters
-- reloid: oid of the table.
-- Return (schema_name, table_name)
-- note that returned names will be quoted if necessary
CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
AS $$
BEGIN
SELECT n.nspname, c.relname
INTO STRICT schema_name, table_name
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = reloid;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Schema and relation names of a table given its reloid
-- Scope: private.
-- Parameters
-- reloid: oid of the table.
-- Return (schema_name, table_name)
-- note that returned names will be quoted if necessary
CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
RETURNS TEXT
AS $$
DECLARE
schema_name TEXT;
BEGIN
SELECT n.nspname
INTO STRICT schema_name
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = reloid;
RETURN schema_name;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;

4
scripts-available/CDB_Quota.sql
View File

@ -35,7 +35,9 @@ BEGIN
table_cat AS (
SELECT
table_name,
EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
EXISTS(select * from raster_tables where o_table_name = table_name)
OR table_name SIMILAR TO _CDB_OverviewTableDiscriminator() || '[\w\d]*'
AS is_overview,
EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
FROM user_tables
),

1
scripts-enabled/085-CDB_OverviewsSupport.sql Symbolic link
View File

@ -0,0 +1 @@
../scripts-available/CDB_OverviewsSupport.sql

0
scripts-enabled/242-CDB_Overviews.sql → scripts-enabled/245-CDB_Overviews.sql
View File

2
test/extension/run_at_cartodb_schema.sql
View File

@ -4,4 +4,4 @@ SET SCHEMA 'cartodb';
\i scripts-available/CDB_ColumnNames.sql
\i scripts-available/CDB_ColumnType.sql
\i scripts-available/CDB_AnalysisCatalog.sql
SET SCHEMA 'public';
SET SCHEMA 'public';

1
test/extension/test.sh
View File

@ -181,6 +181,7 @@ function setup_database() {
log_info "########################### BOOTSTRAP ###########################"
${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
${CMD} -d ${DATABASE} -f scripts-available/CDB_OverviewsSupport.sql
# trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
}