Do not track CDB_ sql function files
This commit is contained in:
parent
dbe033112d
commit
69fa13ab50
1
test/support/.gitignore
vendored
Normal file
1
test/support/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
CDB_*.sql
|
File diff suppressed because it is too large
Load Diff
@ -1,18 +0,0 @@
|
||||
-- Function returning the column names of a table
|
||||
CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
|
||||
RETURNS SETOF information_schema.sql_identifier
|
||||
AS $$
|
||||
|
||||
SELECT c.column_name
|
||||
FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
|
||||
WHERE table_name = _tn.relname
|
||||
AND table_schema = _sn.nspname
|
||||
AND _tn.oid = $1::oid
|
||||
AND _sn.oid = _tn.relnamespace
|
||||
ORDER BY ordinal_position;
|
||||
|
||||
$$ LANGUAGE SQL;
|
||||
|
||||
-- This is to migrate from pre-0.2.0 version
|
||||
-- See http://github.com/CartoDB/cartodb-postgresql/issues/36
|
||||
GRANT EXECUTE ON FUNCTION CDB_ColumnNames(REGCLASS) TO PUBLIC;
|
@ -1,199 +0,0 @@
|
||||
---------------------------
|
||||
-- FDW MANAGEMENT FUNCTIONS
|
||||
--
|
||||
-- All the FDW settings are read from the `cdb_conf.fdws` entry json file.
|
||||
---------------------------
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text, config json)
|
||||
RETURNS void
|
||||
AS $$
|
||||
DECLARE
|
||||
row record;
|
||||
option record;
|
||||
org_role text;
|
||||
BEGIN
|
||||
-- This function tries to be as idempotent as possible, by not creating anything more than once
|
||||
-- (not even using IF NOT EXIST to avoid throwing warnings)
|
||||
IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
|
||||
CREATE EXTENSION postgres_fdw;
|
||||
END IF;
|
||||
-- Create FDW first if it does not exist
|
||||
IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
|
||||
THEN
|
||||
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Set FDW settings
|
||||
FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
|
||||
LOOP
|
||||
IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
|
||||
THEN
|
||||
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
|
||||
ELSE
|
||||
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Create user mappings
|
||||
FOR row IN SELECT p.key, p.value from lateral json_each(config->'users') p LOOP
|
||||
-- Check if entry on pg_user_mappings exists
|
||||
|
||||
IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN
|
||||
EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Update user mapping settings
|
||||
FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP
|
||||
IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN
|
||||
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value);
|
||||
ELSE
|
||||
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
|
||||
-- Create schema if it does not exist.
|
||||
IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
|
||||
EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
|
||||
END IF;
|
||||
|
||||
-- Give the organization role usage permisions over the schema
|
||||
SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() INTO org_role;
|
||||
EXECUTE FORMAT ('GRANT USAGE ON SCHEMA %I TO %I', fdw_name, org_role);
|
||||
|
||||
-- Bring here the remote cdb_tablemetadata
|
||||
IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN
|
||||
EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name);
|
||||
END IF;
|
||||
EXECUTE FORMAT ('GRANT SELECT ON %I.cdb_tablemetadata TO %I', fdw_name, org_role);
|
||||
|
||||
END
|
||||
$$
|
||||
LANGUAGE PLPGSQL;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDWS()
|
||||
RETURNS VOID AS
|
||||
$$
|
||||
DECLARE
|
||||
row record;
|
||||
BEGIN
|
||||
FOR row IN SELECT p.key, p.value from lateral json_each(cartodb.CDB_Conf_GetConf('fdws')) p LOOP
|
||||
EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING row.key, row.value;
|
||||
END LOOP;
|
||||
END
|
||||
$$
|
||||
LANGUAGE PLPGSQL;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text)
|
||||
RETURNS void AS
|
||||
$BODY$
|
||||
DECLARE
|
||||
config json;
|
||||
BEGIN
|
||||
SELECT p.value FROM LATERAL json_each(cartodb.CDB_Conf_GetConf('fdws')) p WHERE p.key = fdw_name INTO config;
|
||||
EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING fdw_name, config;
|
||||
END
|
||||
$BODY$
|
||||
LANGUAGE plpgsql VOLATILE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb.CDB_Add_Remote_Table(source text, table_name text)
|
||||
RETURNS void AS
|
||||
$$
|
||||
BEGIN
|
||||
PERFORM cartodb._CDB_Setup_FDW(source);
|
||||
EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', source, table_name, source, source);
|
||||
--- Grant SELECT to publicuser
|
||||
EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO publicuser;', source, table_name);
|
||||
END
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb.CDB_Get_Foreign_Updated_At(foreign_table regclass)
|
||||
RETURNS timestamp with time zone AS
|
||||
$$
|
||||
DECLARE
|
||||
remote_table_name text;
|
||||
fdw_schema_name text;
|
||||
time timestamp with time zone;
|
||||
BEGIN
|
||||
-- This will turn a local foreign table (referenced as regclass) to its fully qualified text remote table reference.
|
||||
WITH a AS (SELECT ftoptions FROM pg_foreign_table WHERE ftrelid=foreign_table LIMIT 1),
|
||||
b as (SELECT (pg_options_to_table(ftoptions)).* FROM a)
|
||||
SELECT FORMAT('%I.%I', (SELECT option_value FROM b WHERE option_name='schema_name'), (SELECT option_value FROM b WHERE option_name='table_name'))
|
||||
INTO remote_table_name;
|
||||
|
||||
-- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table.
|
||||
SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name;
|
||||
EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
|
||||
RETURN time;
|
||||
END
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid)
|
||||
RETURNS TEXT AS $$
|
||||
SELECT option_value FROM pg_options_to_table((
|
||||
|
||||
SELECT fs.srvoptions
|
||||
FROM pg_foreign_table ft
|
||||
LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid
|
||||
WHERE ft.ftrelid = reloid
|
||||
|
||||
)) WHERE option_name='dbname';
|
||||
$$ LANGUAGE SQL;
|
||||
|
||||
|
||||
-- Return a set of (dbname, schema_name, table_name, updated_at)
|
||||
-- It is aware of foreign tables
|
||||
-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
|
||||
-- Note: dbname is never quoted whereas schema and table names are when needed.
|
||||
CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text)
|
||||
RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz)
|
||||
AS $$
|
||||
WITH query_tables AS (
|
||||
SELECT unnest(CDB_QueryTablesText(query)) schema_table_name
|
||||
), query_tables_oid AS (
|
||||
SELECT schema_table_name, schema_table_name::regclass::oid AS reloid
|
||||
FROM query_tables
|
||||
),
|
||||
fqtn AS (
|
||||
SELECT
|
||||
(CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid)
|
||||
ELSE current_database()
|
||||
END)::text AS dbname,
|
||||
quote_ident(n.nspname::text) schema_name,
|
||||
quote_ident(c.relname::text) table_name,
|
||||
c.relkind,
|
||||
query_tables_oid.reloid
|
||||
FROM query_tables_oid, pg_catalog.pg_class c
|
||||
LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE c.oid = query_tables_oid.reloid
|
||||
)
|
||||
SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name,
|
||||
(CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
|
||||
ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
|
||||
END) AS updated_at
|
||||
FROM fqtn;
|
||||
$$ LANGUAGE SQL;
|
||||
|
||||
|
||||
-- Return the last updated time of a set of tables
|
||||
-- It is aware of foreign tables
|
||||
-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
|
||||
CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[])
|
||||
RETURNS timestamptz AS $$
|
||||
WITH t AS (
|
||||
SELECT unnest(tables) AS schema_table_name
|
||||
), t_oid AS (
|
||||
SELECT (t.schema_table_name)::regclass::oid as reloid FROM t
|
||||
), t_updated_at AS (
|
||||
SELECT
|
||||
(CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
|
||||
ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
|
||||
END) AS updated_at
|
||||
FROM t_oid
|
||||
LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
|
||||
) SELECT max(updated_at) FROM t_updated_at;
|
||||
$$ LANGUAGE SQL;
|
@ -1,683 +0,0 @@
|
||||
-- security definer
|
||||
|
||||
-- Pattern that can be used to detect overview tables and Extract
|
||||
-- the intended zoom level from the table name.
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN '\A_vovw_(\d+)_';
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
-- substring(tablename from _CDB_OverviewTableDiscriminator())
|
||||
|
||||
|
||||
-- Pattern matched by the overview tables of a given base table name.
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN _CDB_OverviewTableDiscriminator() || base_table;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
|
||||
|
||||
-- Name of an overview table, given the base table name and the Z level
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN '_vovw_' || z::text || '_' || base_table;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- Condition to check if a tabla is an overview table of some base table
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
|
||||
RETURNS BOOLEAN
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- Extract the Z level from an overview table name
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
|
||||
RETURNS INTEGER
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- Name of the base table corresponding to an overview table
|
||||
-- Scope: private.
|
||||
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
BEGIN
|
||||
IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
|
||||
RETURN overview_table;
|
||||
ELSE
|
||||
RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
|
||||
END IF;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
|
||||
-- Remove a dataset's existing overview tables.
|
||||
-- Scope: public
|
||||
-- Parameters:
|
||||
-- reloid: oid of the table.
|
||||
CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
|
||||
RETURNS void
|
||||
AS $$
|
||||
DECLARE
|
||||
row record;
|
||||
BEGIN
|
||||
FOR row IN
|
||||
SELECT * FROM CDB_Overviews(reloid)
|
||||
LOOP
|
||||
EXECUTE Format('DROP TABLE %s;', row.overview_table);
|
||||
RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table;
|
||||
END LOOP;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL VOLATILE;
|
||||
|
||||
|
||||
|
||||
-- Return existing overviews (if any) for a given dataset table
|
||||
-- Scope: public
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table.
|
||||
-- Return relation of overviews for the table with
|
||||
-- the base table oid,
|
||||
-- z level of the overview and overview table oid, ordered by z.
|
||||
CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
|
||||
RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
|
||||
AS $$
|
||||
-- FIXME: this will fail if the overview tables
|
||||
-- require a explicit schema name
|
||||
-- possible solutions: return table names as text instead of regclass
|
||||
-- or add schema of reloid before casting to regclass
|
||||
SELECT
|
||||
reloid AS base_table,
|
||||
_CDB_OverviewTableZ(cdb_usertables) AS z,
|
||||
cdb_usertables::regclass AS overview_table
|
||||
FROM CDB_UserTables()
|
||||
WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
|
||||
ORDER BY z;
|
||||
$$ LANGUAGE SQL;
|
||||
|
||||
-- Return existing overviews (if any) for multiple dataset tables.
|
||||
-- Scope: public
|
||||
-- Parameters
|
||||
-- tables: Array of input tables oids
|
||||
-- Return relation of overviews for the table with
|
||||
-- the base table oid,
|
||||
-- z level of the overview and overview table oid, ordered by z.
|
||||
-- Note: CDB_Overviews can be applied to the result of CDB_QueryTablesText
|
||||
-- to obtain the overviews applicable to a query.
|
||||
CREATE OR REPLACE FUNCTION CDB_Overviews(tables regclass[])
|
||||
RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
|
||||
AS $$
|
||||
SELECT
|
||||
base_table::regclass AS base_table,
|
||||
_CDB_OverviewTableZ(cdb_usertables) AS z,
|
||||
cdb_usertables::regclass AS overview_table
|
||||
FROM
|
||||
CDB_UserTables(), unnest(tables) base_table
|
||||
WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
|
||||
ORDER BY base_table, z;
|
||||
$$ LANGUAGE SQL;
|
||||
|
||||
-- Schema and relation names of a table given its reloid
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the table.
|
||||
-- Return (schema_name, table_name)
|
||||
-- note that returned names will be quoted if necessary
|
||||
CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
|
||||
AS $$
|
||||
BEGIN
|
||||
SELECT n.nspname, c.relname
|
||||
INTO STRICT schema_name, table_name
|
||||
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE c.oid = reloid;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- Calculate the estimated extent of a cartodbfy'ed table.
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table.
|
||||
-- Return value A box2d extent in 3857.
|
||||
CREATE OR REPLACE FUNCTION _cdb_estimated_extent(reloid REGCLASS)
|
||||
RETURNS box2d
|
||||
AS $$
|
||||
DECLARE
|
||||
ext box2d;
|
||||
ext_query text;
|
||||
table_id record;
|
||||
BEGIN
|
||||
|
||||
SELECT n.nspname AS schema_name, c.relname table_name INTO STRICT table_id
|
||||
FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
|
||||
|
||||
ext_query = format(
|
||||
'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
|
||||
table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
|
||||
);
|
||||
|
||||
BEGIN
|
||||
EXECUTE ext_query INTO ext;
|
||||
EXCEPTION
|
||||
-- This is the typical ERROR: stats for "mytable" do not exist
|
||||
WHEN internal_error THEN
|
||||
-- Get stats and execute again
|
||||
EXECUTE format('ANALYZE %1$I', reloid);
|
||||
EXECUTE ext_query INTO ext;
|
||||
END;
|
||||
|
||||
RETURN ext;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL VOLATILE;
|
||||
|
||||
-- Determine the max feature density of a given dataset.
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- nz: number of zoom levels to consider from z0 upward.
|
||||
-- Return value: feature density (num_features / webmercator_squared_meters).
|
||||
CREATE OR REPLACE FUNCTION _CDB_Feature_Density(reloid REGCLASS, nz integer)
|
||||
RETURNS FLOAT8
|
||||
AS $$
|
||||
DECLARE
|
||||
fd FLOAT8;
|
||||
min_features TEXT;
|
||||
n integer = 4;
|
||||
c FLOAT8;
|
||||
BEGIN
|
||||
-- TODO: for small total count or extents we could just:
|
||||
-- EXECUTE 'SELECT Count(*)/ST_Area(ST_Extent(the_geom_webmercator)) FROM ' || reloid::text || ';' INTO fd;
|
||||
|
||||
-- min_features is a SQL subexpression which can depend on z and represents
|
||||
-- the minimum number of features to recursively consider a tile.
|
||||
-- We can either use a fixed minimum number of features per tile
|
||||
-- or a minimum feature density by dividing the number of features by
|
||||
-- the area of tiles at level Z: c*c*power(2, -2*z)
|
||||
-- with c = CDB_XYZ_Resolution(-8) (earth circumference)
|
||||
min_features = '500';
|
||||
SELECT CDB_XYZ_Resolution(-8) INTO c;
|
||||
|
||||
-- We first compute a set of *seed* tiles, of the minimum Z level, z0, such that
|
||||
-- they cover the extent of the table and we have at least n of them in each
|
||||
-- linear dimension (i.e. at least n*n tiles cover the extent).
|
||||
-- We compute the number of features in these tiles, and recursively in
|
||||
-- subtiles up to level z0 + nz. Then we compute the maximum of the feature
|
||||
-- density (per tile area in webmercator squared meters) for all the
|
||||
-- considered tiles.
|
||||
EXECUTE Format('
|
||||
WITH RECURSIVE t(x, y, z, e) AS (
|
||||
WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
|
||||
base AS (
|
||||
SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
|
||||
FROM ext
|
||||
),
|
||||
lim AS (
|
||||
SELECT
|
||||
FLOOR((ST_XMin(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x0,
|
||||
FLOOR((ST_XMax(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x1,
|
||||
FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMin(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y1,
|
||||
FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMax(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y0
|
||||
FROM ext, base
|
||||
),
|
||||
seed AS (
|
||||
SELECT xt, yt, base.z, (
|
||||
SELECT count(*) FROM %1$s
|
||||
WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z)
|
||||
) e
|
||||
FROM base, lim, generate_series(lim.x0, lim.x1) xt, generate_series(lim.y0, lim.y1) yt
|
||||
)
|
||||
SELECT * from seed
|
||||
UNION ALL
|
||||
SELECT x*2 + xx, y*2 + yy, t.z+1, (
|
||||
SELECT count(*) FROM %1$s
|
||||
WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
|
||||
)
|
||||
FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
|
||||
WHERE t.e > %2$s AND t.z < (base.z + %3$s)
|
||||
)
|
||||
SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
|
||||
', reloid::text, min_features, nz, n, c, reloid::oid)
|
||||
INTO fd;
|
||||
RETURN fd;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
-- Experimental default strategy to assign a reference base Z level
|
||||
-- to a cartodbfied table. The resulting Z level represents the
|
||||
-- minimum scale level at which the table data can be rendered
|
||||
-- without overcrowded results or loss of detail.
|
||||
-- Parameters:
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- Return value: Z level as an integer
|
||||
CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
|
||||
RETURNS INTEGER
|
||||
AS $$
|
||||
DECLARE
|
||||
lim FLOAT8 := 500; -- TODO: determine/parameterize this
|
||||
nz integer := 4;
|
||||
fd FLOAT8;
|
||||
c FLOAT8;
|
||||
BEGIN
|
||||
-- Compute fd as an estimation of the (maximum) number
|
||||
-- of features per unit of tile area (in webmercator squared meters)
|
||||
SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
|
||||
-- lim maximum number of (desiderable) features per tile
|
||||
-- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
|
||||
-- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
|
||||
-- => fd*ta(z) if the average number of features per tile at level z
|
||||
-- find minimum z so that fd*ta(z) <= lim
|
||||
-- compute a rough 'feature density' value
|
||||
SELECT CDB_XYZ_Resolution(-8) INTO c;
|
||||
RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
-- Overview table name for a given Z level and base dataset or overview table
|
||||
-- Scope: private.
|
||||
-- Parameters:
|
||||
-- ref reference table (can be the base table of the dataset or an existing
|
||||
-- overview) from which the overview is being generated.
|
||||
-- ref_z Z level of the reference table
|
||||
-- overview_z Z level of the overview to be named, must be smaller than ref_z
|
||||
-- Return value: the name to be used for the overview. The name is always
|
||||
-- unqualified (does not include a schema name).
|
||||
CREATE OR REPLACE FUNCTION _CDB_Overview_Name(ref REGCLASS, ref_z INTEGER, overview_z INTEGER)
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
DECLARE
|
||||
schema_name TEXT;
|
||||
base TEXT;
|
||||
suffix TEXT;
|
||||
is_overview BOOLEAN;
|
||||
BEGIN
|
||||
SELECT * FROM _cdb_split_table_name(ref) INTO schema_name, base;
|
||||
SELECT _CDB_OverviewBaseTableName(base) INTO base;
|
||||
RETURN _CDB_OverviewTableName(base, overview_z);
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- Sampling reduction method.
|
||||
-- Valid for any kind of geometry.
|
||||
-- Scope: private.
|
||||
-- reloid original table (can be the base table of the dataset or an existing
|
||||
-- overview) from which the overview is being generated.
|
||||
-- ref_z Z level assigned to the original table
|
||||
-- overview_z Z level of the overview to be generated, must be smaller than ref_z
|
||||
-- Return value: Name of the generated overview table
|
||||
CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
|
||||
RETURNS REGCLASS
|
||||
AS $$
|
||||
DECLARE
|
||||
overview_rel TEXT;
|
||||
fraction FLOAT8;
|
||||
base_name TEXT;
|
||||
class_info RECORD;
|
||||
num_samples INTEGER;
|
||||
BEGIN
|
||||
overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
|
||||
fraction := power(2, 2*(overview_z - ref_z));
|
||||
|
||||
-- FIXME: handle schema name for overview_rel if reloid requires it
|
||||
EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
|
||||
|
||||
-- Estimate number of rows
|
||||
SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
|
||||
WHERE oid = reloid::oid;
|
||||
|
||||
IF class_info.relpages < 2 OR fraction > 0.5 THEN
|
||||
-- We'll avoid possible CDB_RandomTids problems
|
||||
EXECUTE Format('
|
||||
CREATE TABLE %I AS SELECT * FROM %s WHERE random() < %s;
|
||||
', overview_rel, reloid, fraction);
|
||||
ELSE
|
||||
num_samples := ceil(class_info.reltuples*fraction);
|
||||
EXECUTE Format('
|
||||
CREATE TABLE %1$I AS SELECT * FROM %2$s
|
||||
WHERE ctid = ANY (
|
||||
ARRAY[
|
||||
(SELECT CDB_RandomTids(''%2$s'', %3$s))
|
||||
]
|
||||
);
|
||||
', overview_rel, reloid, num_samples);
|
||||
END IF;
|
||||
|
||||
RETURN overview_rel;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
|
||||
-- Register new overview table (post-creation chores)
|
||||
-- Scope: private
|
||||
-- Parameters:
|
||||
-- dataset: oid of the input dataset table, It must be a cartodbfy'ed table.
|
||||
-- overview_table: oid of the overview table to be registered.
|
||||
-- overview_z: intended Z level for the overview table
|
||||
-- This function is declared SECURITY DEFINER so it executes with the privileges
|
||||
-- of the function creator to have a chance to alter the privileges of the
|
||||
-- overview table to match those of the dataset. It will only perform any change
|
||||
-- if the overview table belgons to the same scheme as the dataset and it
|
||||
-- matches the scheme naming for overview tables.
|
||||
CREATE OR REPLACE FUNCTION _CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
|
||||
RETURNS VOID
|
||||
AS $$
|
||||
DECLARE
|
||||
sql TEXT;
|
||||
table_owner TEXT;
|
||||
dataset_scheme TEXT;
|
||||
dataset_name TEXT;
|
||||
overview_scheme TEXT;
|
||||
overview_name TEXT;
|
||||
BEGIN
|
||||
-- This function will only register a table as an overview table if it matches
|
||||
-- the overviews naming scheme for the dataset and z level and the table belongs
|
||||
-- to the same scheme as the the dataset
|
||||
SELECT * FROM _cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name;
|
||||
SELECT * FROM _cdb_split_table_name(overview_table) INTO overview_scheme, overview_name;
|
||||
IF dataset_scheme = overview_scheme AND
|
||||
overview_name = _CDB_OverviewTableName(dataset_name, overview_z) THEN
|
||||
|
||||
-- preserve the owner of the base table
|
||||
SELECT u.usename
|
||||
FROM pg_catalog.pg_class c JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
|
||||
WHERE c.relname = dataset::text
|
||||
INTO table_owner;
|
||||
EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
|
||||
|
||||
-- preserve the table privileges
|
||||
UPDATE pg_class c_to
|
||||
SET relacl = c_from.relacl
|
||||
FROM pg_class c_from
|
||||
WHERE c_from.oid = dataset
|
||||
AND c_to.oid = overview_table;
|
||||
|
||||
PERFORM _CDB_Add_Indexes(overview_table);
|
||||
|
||||
-- TODO: If metadata about existing overviews is to be stored
|
||||
-- it should be done here (CDB_Overviews would consume such metadata)
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL SECURITY DEFINER;
|
||||
|
||||
-- Dataset attributes (column names other than the
|
||||
-- CartoDB primary key and geometry columns) which should be aggregated
|
||||
-- in aggregated overviews.
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- Return value: set of attribute names
|
||||
CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes(reloid REGCLASS)
|
||||
RETURNS SETOF information_schema.sql_identifier
|
||||
AS $$
|
||||
SELECT c FROM CDB_ColumnNames(reloid) c, _CDB_Columns() cdb
|
||||
WHERE c NOT IN (
|
||||
cdb.pkey, cdb.geomcol, cdb.mercgeomcol
|
||||
)
|
||||
$$ LANGUAGE SQL STABLE;
|
||||
|
||||
-- List of dataset attributes to be aggregated in aggregated overview
|
||||
-- as a comma-separated SQL expression.
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- Return value: SQL subexpression as text
|
||||
CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes_Expression(reloid REGCLASS)
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
DECLARE
|
||||
attr_list TEXT;
|
||||
BEGIN
|
||||
SELECT string_agg(s.c, ',') FROM (
|
||||
SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
|
||||
) AS s INTO attr_list;
|
||||
|
||||
RETURN attr_list;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
-- SQL Aggregation expression for a datase attribute
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- column_name: column to be aggregated
|
||||
-- table_alias: (optional) table qualifier for the column to be aggregated
|
||||
-- Return SQL subexpression as text with aggregated attribute aliased
|
||||
-- with its original name.
|
||||
CREATE OR REPLACE FUNCTION _CDB_Attribute_Aggregation_Expression(reloid REGCLASS, column_name TEXT, table_alias TEXT DEFAULT '')
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
DECLARE
|
||||
column_type TEXT;
|
||||
qualified_column TEXT;
|
||||
BEGIN
|
||||
IF table_alias <> '' THEN
|
||||
qualified_column := Format('%I.%I', table_alias, column_name);
|
||||
ELSE
|
||||
qualified_column := Format('%I', column_name);
|
||||
END IF;
|
||||
|
||||
column_type := CDB_ColumnType(reloid, column_name);
|
||||
|
||||
CASE column_type
|
||||
WHEN 'double precision', 'real', 'integer', 'bigint' THEN
|
||||
RETURN Format('AVG(%s)::' || column_type, qualified_column);
|
||||
WHEN 'text' THEN
|
||||
-- TODO: we could define a new aggregate function that returns distinct
|
||||
-- separated values with a limit, adding ellipsis if more values existed
|
||||
-- e.g. with '/' as separator and a limit of three:
|
||||
-- 'A', 'B', 'A', 'C', 'D' => 'A/B/C/...'
|
||||
-- Other ideas: if value is unique then use it, otherwise use something
|
||||
-- like '*' or '(varies)' or '(multiple values)', or NULL
|
||||
-- Using 'string_agg(' || qualified_column || ',''/'')'
|
||||
-- here causes
|
||||
RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
|
||||
ELSE
|
||||
RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
|
||||
END CASE;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL IMMUTABLE;
|
||||
|
||||
-- List of dataset aggregated attributes as a comma-separated SQL expression.
|
||||
-- Scope: private.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- table_alias: (optional) table qualifier for the columns to be aggregated
|
||||
-- Return value: SQL subexpression as text
|
||||
CREATE OR REPLACE FUNCTION _CDB_Aggregated_Attributes_Expression(reloid REGCLASS, table_alias TEXT DEFAULT '')
|
||||
RETURNS TEXT
|
||||
AS $$
|
||||
DECLARE
|
||||
attr_list TEXT;
|
||||
BEGIN
|
||||
SELECT string_agg(_CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
|
||||
FROM (
|
||||
SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
|
||||
) AS s INTO attr_list;
|
||||
|
||||
RETURN attr_list;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
-- Array of geometry types detected in a cartodbfied table
|
||||
-- For effciency only look at a limited number of rwos.
|
||||
-- Parameters
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
|
||||
-- Return value: array of geometry type names
|
||||
CREATE OR REPLACE FUNCTION _CDB_GeometryTypes(reloid REGCLASS)
|
||||
RETURNS TEXT[]
|
||||
AS $$
|
||||
DECLARE
|
||||
gtypes TEXT[];
|
||||
BEGIN
|
||||
EXECUTE Format('
|
||||
SELECT array_agg(DISTINCT ST_GeometryType(the_geom)) FROM (
|
||||
SELECT the_geom FROM %s
|
||||
WHERE (the_geom is not null) LIMIT 10
|
||||
) as geom_types
|
||||
', reloid)
|
||||
INTO gtypes;
|
||||
RETURN gtypes;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL STABLE;
|
||||
|
||||
-- Experimental Overview reduction method for point datasets.
|
||||
-- It clusters the points using a grid, then aggregates the point in each
|
||||
-- cluster into a point at the centroid of the clustered records.
|
||||
-- Scope: private.
|
||||
-- Parameters:
|
||||
-- reloid original table (can be the base table of the dataset or an existing
|
||||
-- overview) from which the overview is being generated.
|
||||
-- ref_z Z level assigned to the original table
|
||||
-- overview_z Z level of the overview to be generated, must be smaller than ref_z
|
||||
-- Return value: Name of the generated overview table
|
||||
CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
|
||||
RETURNS REGCLASS
|
||||
AS $$
|
||||
DECLARE
|
||||
overview_rel TEXT;
|
||||
reduction FLOAT8;
|
||||
base_name TEXT;
|
||||
grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
|
||||
grid_m FLOAT8;
|
||||
aggr_attributes TEXT;
|
||||
attributes TEXT;
|
||||
columns TEXT;
|
||||
gtypes TEXT[];
|
||||
BEGIN
|
||||
SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
|
||||
IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
|
||||
-- This strategy only supports datasets with point geomety
|
||||
RETURN NULL;
|
||||
RETURN 'x';
|
||||
END IF;
|
||||
|
||||
--TODO: check applicability: geometry type, minimum number of points...
|
||||
|
||||
overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
|
||||
|
||||
-- compute grid cell size using the overview_z dimension...
|
||||
SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
|
||||
|
||||
attributes := _CDB_Aggregable_Attributes_Expression(reloid);
|
||||
aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
|
||||
IF attributes <> '' THEN
|
||||
attributes := ', ' || attributes;
|
||||
END IF;
|
||||
IF aggr_attributes <> '' THEN
|
||||
aggr_attributes := aggr_attributes || ', ';
|
||||
END IF;
|
||||
|
||||
-- compute the resulting columns in the same order as in the base table
|
||||
-- cartodb_id,
|
||||
-- ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom,
|
||||
-- ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator
|
||||
-- %4$s
|
||||
WITH cols AS (
|
||||
SELECT
|
||||
CASE c
|
||||
WHEN 'cartodb_id' THEN 'cartodb_id'
|
||||
WHEN 'the_geom' THEN
|
||||
'ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom'
|
||||
WHEN 'the_geom_webmercator' THEN
|
||||
'ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator'
|
||||
ELSE c
|
||||
END AS column
|
||||
FROM CDB_ColumnNames(reloid) c
|
||||
)
|
||||
SELECT string_agg(s.column, ',') FROM (
|
||||
SELECT * FROM cols
|
||||
) AS s INTO columns;
|
||||
|
||||
-- FIXME: handle schema name for overview_rel if reloid requires it
|
||||
EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
|
||||
|
||||
-- Now we cluster the data using a grid of size grid_m
|
||||
-- and selecte the centroid (average coordinates) of each cluster.
|
||||
-- If we had a selected numeric attribute of interest we could use it
|
||||
-- as a weight for the average coordinates.
|
||||
EXECUTE Format('
|
||||
CREATE TABLE %3$I AS
|
||||
WITH clusters AS (
|
||||
SELECT
|
||||
%5$s
|
||||
count(*) AS n,
|
||||
SUM(ST_X(f.the_geom_webmercator)) AS sx,
|
||||
SUM(ST_Y(f.the_geom_webmercator)) AS sy,
|
||||
Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
|
||||
Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
|
||||
MIN(cartodb_id) AS cartodb_id
|
||||
FROM %1$s f
|
||||
GROUP BY gx, gy
|
||||
)
|
||||
SELECT %6$s FROM clusters
|
||||
', reloid::text, grid_m, overview_rel, attributes, aggr_attributes, columns);
|
||||
|
||||
RETURN overview_rel;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
|
||||
-- Create overview tables for a dataset.
|
||||
-- Scope: public
|
||||
-- Parameters:
|
||||
-- reloid: oid of the input table. It must be a cartodbfy'ed table with
|
||||
-- vector features.
|
||||
-- refscale_strategy: function that computes the reference Z of the dataset
|
||||
-- reduce_strategy: function that generates overviews from a base table
|
||||
-- or higher level overview. The overview tables
|
||||
-- created by the strategy must have the same columns
|
||||
-- as the base table and in the same order.
|
||||
-- Return value: Array with the names of the generated overview tables
|
||||
CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
|
||||
RETURNS text[]
|
||||
AS $$
|
||||
DECLARE
|
||||
ref_z integer;
|
||||
overviews_z integer[];
|
||||
base_z integer;
|
||||
base_rel REGCLASS;
|
||||
overview_z integer;
|
||||
overview_tables REGCLASS[];
|
||||
overviews_step integer := 1;
|
||||
BEGIN
|
||||
-- Determine the referece zoom level
|
||||
EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
|
||||
|
||||
-- Determine overlay zoom levels
|
||||
-- TODO: should be handled by the refscale_strategy?
|
||||
overview_z := ref_z - 1;
|
||||
WHILE overview_z >= 0 LOOP
|
||||
SELECT array_append(overviews_z, overview_z) INTO overviews_z;
|
||||
overview_z := overview_z - overviews_step;
|
||||
END LOOP;
|
||||
|
||||
-- Create overlay tables
|
||||
base_z := ref_z;
|
||||
base_rel := reloid;
|
||||
FOREACH overview_z IN ARRAY overviews_z LOOP
|
||||
EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
|
||||
IF base_rel IS NULL THEN
|
||||
EXIT;
|
||||
END IF;
|
||||
base_z := overview_z;
|
||||
PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
|
||||
SELECT array_append(overview_tables, base_rel) INTO overview_tables;
|
||||
END LOOP;
|
||||
|
||||
RETURN overview_tables;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
@ -1,14 +0,0 @@
|
||||
-- Return an array of statements found in the given query text
|
||||
--
|
||||
-- Regexp curtesy of Hubert Lubaczewski (depesz)
|
||||
-- Implemented in plpython for performance reasons
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
|
||||
RETURNS SETOF TEXT AS $$
|
||||
import re
|
||||
pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
|
||||
for match in pat.findall(query):
|
||||
cleaned = match[0].strip()
|
||||
if ( cleaned ):
|
||||
yield cleaned
|
||||
$$ language 'plpythonu' IMMUTABLE STRICT;
|
@ -1,78 +0,0 @@
|
||||
-- Return an array of table names scanned by a given query
|
||||
--
|
||||
-- Requires PostgreSQL 9.x+
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
|
||||
RETURNS text[]
|
||||
AS $$
|
||||
DECLARE
|
||||
exp XML;
|
||||
tables text[];
|
||||
rec RECORD;
|
||||
rec2 RECORD;
|
||||
BEGIN
|
||||
|
||||
tables := '{}';
|
||||
|
||||
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
|
||||
|
||||
IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
|
||||
--RAISE WARNING 'Skipping %', rec.q;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
|
||||
EXCEPTION WHEN others THEN
|
||||
-- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
|
||||
-- the affected table ?
|
||||
RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
|
||||
RAISE EXCEPTION '%', SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
|
||||
-- Now need to extract all values of <Relation-Name>
|
||||
|
||||
-- RAISE DEBUG 'Explain: %', exp;
|
||||
|
||||
FOR rec2 IN WITH
|
||||
inp AS (
|
||||
SELECT
|
||||
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
|
||||
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
|
||||
)
|
||||
SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
|
||||
LOOP
|
||||
-- RAISE DEBUG 'tab: %', rec2.p;
|
||||
-- RAISE DEBUG 'sc: %', rec2.sc;
|
||||
tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
|
||||
END LOOP;
|
||||
|
||||
-- RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
END LOOP;
|
||||
|
||||
-- RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
-- Remove duplicates and sort by name
|
||||
IF array_upper(tables, 1) > 0 THEN
|
||||
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
|
||||
SELECT array_agg(p) from dist into tables;
|
||||
END IF;
|
||||
|
||||
--RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
return tables;
|
||||
END
|
||||
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
|
||||
|
||||
|
||||
-- Keep CDB_QueryTables with same signature for backwards compatibility.
|
||||
-- It should probably be removed in the future.
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
|
||||
RETURNS name[]
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN CDB_QueryTablesText(query)::name[];
|
||||
END
|
||||
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
|
@ -1,143 +0,0 @@
|
||||
|
||||
CREATE TABLE IF NOT EXISTS
|
||||
public.CDB_TableMetadata (
|
||||
tabname regclass not null primary key,
|
||||
updated_at timestamp with time zone not null default now()
|
||||
);
|
||||
|
||||
CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS
|
||||
SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at
|
||||
FROM public.CDB_TableMetadata, pg_catalog.pg_class c
|
||||
LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid;
|
||||
|
||||
-- No one can see this
|
||||
-- Updates are only possible trough the security definer trigger
|
||||
-- GRANT SELECT ON public.CDB_TableMetadata TO public;
|
||||
|
||||
--
|
||||
-- Trigger logging updated_at in the CDB_TableMetadata
|
||||
-- and notifying cdb_tabledata_update with table name as payload.
|
||||
--
|
||||
-- Attach to tables like this:
|
||||
--
|
||||
-- CREATE trigger track_updates
|
||||
-- AFTER INSERT OR UPDATE OR TRUNCATE OR DELETE ON <tablename>
|
||||
-- FOR EACH STATEMENT
|
||||
-- EXECUTE PROCEDURE cdb_tablemetadata_trigger();
|
||||
--
|
||||
-- NOTE: _never_ attach to CDB_TableMetadata ...
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_TableMetadata_Trigger()
|
||||
RETURNS trigger AS
|
||||
$$
|
||||
BEGIN
|
||||
-- Guard against infinite loop
|
||||
IF TG_RELID = 'public.CDB_TableMetadata'::regclass::oid THEN
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
-- Cleanup stale entries
|
||||
DELETE FROM public.CDB_TableMetadata
|
||||
WHERE NOT EXISTS (
|
||||
SELECT oid FROM pg_class WHERE oid = tabname
|
||||
);
|
||||
|
||||
WITH nv as (
|
||||
SELECT TG_RELID as tabname, NOW() as t
|
||||
), updated as (
|
||||
UPDATE public.CDB_TableMetadata x SET updated_at = nv.t
|
||||
FROM nv WHERE x.tabname = nv.tabname
|
||||
RETURNING x.tabname
|
||||
)
|
||||
INSERT INTO public.CDB_TableMetadata SELECT nv.*
|
||||
FROM nv LEFT JOIN updated USING(tabname)
|
||||
WHERE updated.tabname IS NULL;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$
|
||||
LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
|
||||
|
||||
--
|
||||
-- Trigger invalidating varnish whenever CDB_TableMetadata
|
||||
-- record change.
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION _CDB_TableMetadata_Updated()
|
||||
RETURNS trigger AS
|
||||
$$
|
||||
DECLARE
|
||||
tabname TEXT;
|
||||
rec RECORD;
|
||||
found BOOL;
|
||||
BEGIN
|
||||
|
||||
IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
|
||||
tabname = NEW.tabname;
|
||||
ELSE
|
||||
tabname = OLD.tabname;
|
||||
END IF;
|
||||
|
||||
-- Notify table data update
|
||||
-- This needs a little bit more of research regarding security issues
|
||||
-- see https://github.com/CartoDB/cartodb/pull/241
|
||||
-- PERFORM pg_notify('cdb_tabledata_update', tabname);
|
||||
|
||||
--RAISE NOTICE 'Table % was updated', tabname;
|
||||
|
||||
-- This will be needed until we'll have someone listening
|
||||
-- on the event we just broadcasted:
|
||||
--
|
||||
-- LISTEN cdb_tabledata_update;
|
||||
--
|
||||
|
||||
-- Call the first varnish invalidation function owned
|
||||
-- by a superuser found in cartodb or public schema
|
||||
-- (in that order)
|
||||
found := false;
|
||||
FOR rec IN SELECT u.usesuper, u.usename, n.nspname, p.proname
|
||||
FROM pg_proc p, pg_namespace n, pg_user u
|
||||
WHERE p.proname = 'cdb_invalidate_varnish'
|
||||
AND p.pronamespace = n.oid
|
||||
AND n.nspname IN ('public', 'cartodb')
|
||||
AND u.usesysid = p.proowner
|
||||
AND u.usesuper
|
||||
ORDER BY n.nspname
|
||||
LOOP
|
||||
EXECUTE 'SELECT ' || quote_ident(rec.nspname) || '.'
|
||||
|| quote_ident(rec.proname)
|
||||
|| '(' || quote_literal(tabname) || ')';
|
||||
found := true;
|
||||
EXIT;
|
||||
END LOOP;
|
||||
IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$
|
||||
LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS table_modified ON CDB_TableMetadata;
|
||||
-- NOTE: on DELETE we would be unable to convert the table
|
||||
-- oid (regclass) to its name
|
||||
CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
|
||||
ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
|
||||
_CDB_TableMetadata_Updated();
|
||||
|
||||
|
||||
-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata
|
||||
CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass)
|
||||
RETURNS void AS
|
||||
$$
|
||||
BEGIN
|
||||
WITH upsert AS (
|
||||
UPDATE public.cdb_tablemetadata
|
||||
SET updated_at = NOW()
|
||||
WHERE tabname = tablename
|
||||
RETURNING *
|
||||
)
|
||||
INSERT INTO public.cdb_tablemetadata (tabname, updated_at)
|
||||
SELECT tablename, NOW()
|
||||
WHERE NOT EXISTS (SELECT * FROM upsert);
|
||||
END;
|
||||
$$
|
||||
LANGUAGE 'plpgsql' VOLATILE STRICT;
|
@ -1,28 +0,0 @@
|
||||
-- Function returning list of cartodb user tables
|
||||
--
|
||||
-- The optional argument restricts the result to tables
|
||||
-- of the specified access type.
|
||||
--
|
||||
-- Currently accepted permissions are: 'public', 'private' or 'all'
|
||||
--
|
||||
DROP FUNCTION IF EXISTS CDB_UserTables(text);
|
||||
CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
|
||||
RETURNS SETOF name
|
||||
AS $$
|
||||
|
||||
SELECT c.relname
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind = 'r'
|
||||
AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
|
||||
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
|
||||
AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
|
||||
WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
|
||||
WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')
|
||||
ELSE false END;
|
||||
|
||||
$$ LANGUAGE 'sql';
|
||||
|
||||
-- This is to migrate from pre-0.2.0 version
|
||||
-- See http://github.com/CartoDB/cartodb-postgresql/issues/36
|
||||
GRANT EXECUTE ON FUNCTION CDB_UserTables(text) TO public;
|
@ -1,30 +0,0 @@
|
||||
CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
|
||||
BEGIN
|
||||
CASE
|
||||
WHEN scaleDenominator > 500000000 THEN RETURN 0;
|
||||
WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
|
||||
WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
|
||||
WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
|
||||
WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
|
||||
WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
|
||||
WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
|
||||
WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
|
||||
WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
|
||||
WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
|
||||
WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
|
||||
WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
|
||||
WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
|
||||
WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
|
||||
WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
|
||||
WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
|
||||
WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
|
||||
WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
|
||||
WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
|
||||
WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
|
||||
WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
|
||||
WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
|
||||
WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
|
||||
WHEN scaleDenominator <= 100 THEN RETURN 23;
|
||||
END CASE;
|
||||
END
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
Loading…
Reference in New Issue
Block a user