cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use node-cartodb-querytables library

Browse Source
This commit is contained in:
Alejandro Martínez 2016-02-22 12:21:44 +01:00
parent 028e0d1553
commit 5e06711b4b
14 changed files with 2444 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/app.js
View File

@ -29,7 +29,6 @@ var UserIndexer = require('../batch/user_indexer');
var JobBackend = require('../batch/job_backend'); var JobBackend = require('../batch/job_backend');
var JobCanceller = require('../batch/job_canceller'); var JobCanceller = require('../batch/job_canceller');
var UserDatabaseMetadataService = require('../batch/user_database_metadata_service'); var UserDatabaseMetadataService = require('../batch/user_database_metadata_service');
var QueryTablesApi = require('./services/query-tables-api');
var cors = require('./middlewares/cors'); var cors = require('./middlewares/cors');
@ -71,7 +70,6 @@ function App() {
// consider entries expired after these many milliseconds (10 minutes by default) // consider entries expired after these many milliseconds (10 minutes by default)
maxAge: global.settings.tableCacheMaxAge || 1000*60*10 maxAge: global.settings.tableCacheMaxAge || 1000*60*10
}); });
var queryTablesApi = new QueryTablesApi(tableCache);
// Size based on https://github.com/CartoDB/cartodb.js/blob/3.15.2/src/geo/layer_definition.js#L72 // Size based on https://github.com/CartoDB/cartodb.js/blob/3.15.2/src/geo/layer_definition.js#L72
var SQL_QUERY_BODY_LOG_MAX_LENGTH = 2000; var SQL_QUERY_BODY_LOG_MAX_LENGTH = 2000;
@ -190,7 +188,7 @@ function App() {
var genericController = new GenericController(); var genericController = new GenericController();
genericController.route(app); genericController.route(app);
var queryController = new QueryController(userDatabaseService, queryTablesApi, statsd_client); var queryController = new QueryController(userDatabaseService, statsd_client);
queryController.route(app); queryController.route(app);
var jobController = new JobController(userDatabaseService, jobBackend, jobCanceller); var jobController = new JobController(userDatabaseService, jobBackend, jobCanceller);

30
app/controllers/query_controller.js
View File

@ -4,23 +4,22 @@ var _ = require('underscore');
var step = require('step'); var step = require('step');
var assert = require('assert'); var assert = require('assert');
var PSQL = require('cartodb-psql'); var PSQL = require('cartodb-psql');
var QueryTables = require('node-cartodb-query-tables');
var AuthApi = require('../auth/auth_api'); var AuthApi = require('../auth/auth_api');
var queryMayWrite = require('../utils/query_may_write');
var CdbRequest = require('../models/cartodb_request'); var CdbRequest = require('../models/cartodb_request');
var formats = require('../models/formats'); var formats = require('../models/formats');
var sanitize_filename = require('../utils/filename_sanitizer'); var sanitize_filename = require('../utils/filename_sanitizer');
var getContentDisposition = require('../utils/content_disposition'); var getContentDisposition = require('../utils/content_disposition');
var generateCacheKey = require('../utils/cache_key_generator');
var handleException = require('../utils/error_handler'); var handleException = require('../utils/error_handler');
var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default
var cdbReq = new CdbRequest(); var cdbReq = new CdbRequest();
function QueryController(userDatabaseService, queryTablesApi, statsd_client) { function QueryController(userDatabaseService, statsd_client) {
this.queryTablesApi = queryTablesApi;
this.statsd_client = statsd_client; this.statsd_client = statsd_client;
this.userDatabaseService = userDatabaseService; this.userDatabaseService = userDatabaseService;
} }
@ -133,22 +132,22 @@ QueryController.prototype.handleQuery = function (req, res) {
checkAborted('queryExplain'); checkAborted('queryExplain');
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(authDbParams, sql, this); var pg = new PSQL(authDbParams, {}, { destroyOnError: true });
QueryTables.getAffectedTablesFromQuery(pg, sql, this);
}, },
function setHeaders(err, queryExplainResult) { function setHeaders(err, affectedTables) {
assert.ifError(err); assert.ifError(err);
var mayWrite = queryMayWrite(sql);
if ( req.profiler ) { if ( req.profiler ) {
req.profiler.done('queryExplain'); req.profiler.done('queryExplain');
} }
checkAborted('setHeaders'); checkAborted('setHeaders');
if (!dbopts.authenticated) { if (!dbopts.authenticated) {
var affected_tables = queryExplainResult.affectedTables; for ( var i = 0; i < affectedTables.tables.length; ++i ) {
for ( var i = 0; i < affected_tables.length; ++i ) { var t = affectedTables.tables[i];
var t = affected_tables[i]; if ( t.table_name.match(/\bpg_/) ) {
if ( t.match(/\bpg_/) ) {
var e = new SyntaxError("system tables are forbidden"); var e = new SyntaxError("system tables are forbidden");
e.http_status = 403; e.http_status = 403;
throw(e); throw(e);
@ -171,16 +170,17 @@ QueryController.prototype.handleQuery = function (req, res) {
if (cachePolicy === 'persist') { if (cachePolicy === 'persist') {
res.header('Cache-Control', 'public,max-age=' + ONE_YEAR_IN_SECONDS); res.header('Cache-Control', 'public,max-age=' + ONE_YEAR_IN_SECONDS);
} else { } else {
var maxAge = (queryExplainResult.mayWrite) ? 0 : ONE_YEAR_IN_SECONDS; var maxAge = (mayWrite) ? 0 : ONE_YEAR_IN_SECONDS;
res.header('Cache-Control', 'no-cache,max-age='+maxAge+',must-revalidate,public'); res.header('Cache-Control', 'no-cache,max-age='+maxAge+',must-revalidate,public');
} }
// Only set an X-Cache-Channel for responses we want Varnish to cache. // Only set an X-Cache-Channel for responses we want Varnish to cache.
if (queryExplainResult.affectedTables.length > 0 && !queryExplainResult.mayWrite) { if (affectedTables.tables.length > 0 && !mayWrite) {
res.header('X-Cache-Channel', generateCacheKey(dbopts.dbname, queryExplainResult.affectedTables)); res.header('X-Cache-Channel', affectedTables.getCacheChannel());
res.header('Surrogate-Key', affectedTables.key());
} }
res.header('Last-Modified', new Date(queryExplainResult.lastModified).toUTCString()); res.header('Last-Modified', new Date(affectedTables.getLastUpdatedAt()).toUTCString());
return null; return null;
}, },

112
app/services/query-tables-api.js
View File

@ -1,112 +0,0 @@
var PSQL = require('cartodb-psql');
var generateMD5 = require('../utils/md5');
var queryMayWrite = require('../utils/query_may_write');
function QueryTablesApi(tableCache) {
this.tableCache = tableCache;
}
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (connectionParams, sql, callback) {
var self = this;
var cacheKey = sqlCacheKey(connectionParams.user, sql);
var queryExplainResult = this.tableCache.get(cacheKey);
if (queryExplainResult) {
queryExplainResult.hits++;
getLastUpdatedTime(connectionParams, queryExplainResult.affectedTables, function(err, lastUpdatedTime) {
return callback(null, {
affectedTables: queryExplainResult.affectedTables,
lastModified: lastUpdatedTime,
mayWrite: queryExplainResult.mayWrite
});
});
} else {
getAffectedTablesAndLastUpdatedTime(connectionParams, sql, function(err, affectedTablesAndLastUpdatedTime) {
var queryExplainResult = {
affectedTables: affectedTablesAndLastUpdatedTime.affectedTables,
mayWrite: queryMayWrite(sql),
hits: 1
};
self.tableCache.set(cacheKey, queryExplainResult);
return callback(null, {
affectedTables: queryExplainResult.affectedTables,
lastModified: affectedTablesAndLastUpdatedTime.lastUpdatedTime,
mayWrite: queryExplainResult.mayWrite
});
});
}
};
function getAffectedTablesAndLastUpdatedTime(connectionParams, sql, callback) {
var query = [
'WITH querytables AS (',
'SELECT * FROM CDB_QueryTablesText($quotesql$' + sql + '$quotesql$) as tablenames',
')',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
var pg = new PSQL(connectionParams, {}, { destroyOnError: true });
pg.query(query, function handleAffectedTablesAndLastUpdatedTimeRows(err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
logIfError(err, sql, rows);
var result = rows[0] || {};
// This is an Array, so no need to split into parts
var tableNames = result.tablenames || [];
var lastUpdatedTime = (Number.isFinite(result.max)) ? (result.max * 1000) : Date.now();
return callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime
});
}, true);
}
function getLastUpdatedTime(connectionParams, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, Date.now());
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
var pg = new PSQL(connectionParams, {}, { destroyOnError: true });
pg.query(query, function handleLastUpdatedTimeRows (err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
var result = rows[0] || {};
var lastUpdatedTime = (Number.isFinite(result.max)) ? (result.max * 1000) : Date.now();
return callback(null, lastUpdatedTime);
}, true);
}
function logIfError(err, sql, rows) {
if (err || rows.length !== 1) {
var errorMessage = (err && err.message) || 'unknown error';
console.error("Error on query explain '%s': %s", sql, errorMessage);
}
}
function sqlCacheKey(user, sql) {
return user + ':' + generateMD5(sql);
}

3
package.json
View File

@ -31,7 +31,8 @@
"step-profiler": "~0.1.0", "step-profiler": "~0.1.0",
"topojson": "0.0.8", "topojson": "0.0.8",
"underscore": "~1.6.0", "underscore": "~1.6.0",
"queue-async": "~1.0.7" "queue-async": "~1.0.7",
"node-cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master"
}, },
"devDependencies": { "devDependencies": {
"istanbul": "~0.4.2", "istanbul": "~0.4.2",

2
test/acceptance/app.test.js
View File

@ -789,7 +789,7 @@ it('DROP TABLE with GET and auth', function(done){
}); });
}); });
test('CREATE FUNCTION with GET and auth', function(done){ it('CREATE FUNCTION with GET and auth', function(done){
assert.response(app, { assert.response(app, {
url: "/api/v1/sql?" + querystring.stringify({ url: "/api/v1/sql?" + querystring.stringify({
q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'', q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'',

10
test/prepare_db.sh
View File

@ -85,10 +85,12 @@ if test x"$PREPARE_PGSQL" = xyes; then
# TODO: send in a single run, togheter with test.sql # TODO: send in a single run, togheter with test.sql
psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB}
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o support/CDB_QueryStatements.sql for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o support/CDB_QueryTables.sql do
psql -f support/CDB_QueryStatements.sql ${TEST_DB} curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o support/$i.sql
psql -f support/CDB_QueryTables.sql ${TEST_DB} cat support/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
| psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
done
fi fi

2
test/run_tests.sh
View File

@ -135,7 +135,7 @@ echo
if test x"$OPT_COVERAGE" = xyes; then if test x"$OPT_COVERAGE" = xyes; then
echo "Running tests with coverage" echo "Running tests with coverage"
./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd --trace -t 5000 ${TESTS}
else else
echo "Running tests" echo "Running tests"
mocha -u tdd -t 5000 ${TESTS} mocha -u tdd -t 5000 ${TESTS}

1317
test/support/CDB_CartodbfyTable.sql Normal file
View File

File diff suppressed because it is too large Load Diff

18
test/support/CDB_ColumnNames.sql Normal file
View File

@ -0,0 +1,18 @@
-- Function returning the column names of a table
CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
RETURNS SETOF information_schema.sql_identifier
AS $$
SELECT c.column_name
FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
WHERE table_name = _tn.relname
AND table_schema = _sn.nspname
AND _tn.oid = $1::oid
AND _sn.oid = _tn.relnamespace
ORDER BY ordinal_position;
$$ LANGUAGE SQL;
-- This is to migrate from pre-0.2.0 version
-- See http://github.com/CartoDB/cartodb-postgresql/issues/36
GRANT EXECUTE ON FUNCTION CDB_ColumnNames(REGCLASS) TO PUBLIC;

199
test/support/CDB_ForeignTable.sql Normal file
View File

@ -0,0 +1,199 @@
---------------------------
-- FDW MANAGEMENT FUNCTIONS
--
-- All the FDW settings are read from the `cdb_conf.fdws` entry json file.
---------------------------
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text, config json)
RETURNS void
AS $$
DECLARE
row record;
option record;
org_role text;
BEGIN
-- This function tries to be as idempotent as possible, by not creating anything more than once
-- (not even using IF NOT EXIST to avoid throwing warnings)
IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
CREATE EXTENSION postgres_fdw;
END IF;
-- Create FDW first if it does not exist
IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
THEN
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
END IF;
-- Set FDW settings
FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
THEN
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
ELSE
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
END IF;
END LOOP;
-- Create user mappings
FOR row IN SELECT p.key, p.value from lateral json_each(config->'users') p LOOP
-- Check if entry on pg_user_mappings exists
IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN
EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name);
END IF;
-- Update user mapping settings
FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value);
ELSE
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value);
END IF;
END LOOP;
END LOOP;
-- Create schema if it does not exist.
IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
END IF;
-- Give the organization role usage permisions over the schema
SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() INTO org_role;
EXECUTE FORMAT ('GRANT USAGE ON SCHEMA %I TO %I', fdw_name, org_role);
-- Bring here the remote cdb_tablemetadata
IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN
EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name);
END IF;
EXECUTE FORMAT ('GRANT SELECT ON %I.cdb_tablemetadata TO %I', fdw_name, org_role);
END
$$
LANGUAGE PLPGSQL;
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDWS()
RETURNS VOID AS
$$
DECLARE
row record;
BEGIN
FOR row IN SELECT p.key, p.value from lateral json_each(cartodb.CDB_Conf_GetConf('fdws')) p LOOP
EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING row.key, row.value;
END LOOP;
END
$$
LANGUAGE PLPGSQL;
CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text)
RETURNS void AS
$BODY$
DECLARE
config json;
BEGIN
SELECT p.value FROM LATERAL json_each(cartodb.CDB_Conf_GetConf('fdws')) p WHERE p.key = fdw_name INTO config;
EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING fdw_name, config;
END
$BODY$
LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION cartodb.CDB_Add_Remote_Table(source text, table_name text)
RETURNS void AS
$$
BEGIN
PERFORM cartodb._CDB_Setup_FDW(source);
EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', source, table_name, source, source);
--- Grant SELECT to publicuser
EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO publicuser;', source, table_name);
END
$$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cartodb.CDB_Get_Foreign_Updated_At(foreign_table regclass)
RETURNS timestamp with time zone AS
$$
DECLARE
remote_table_name text;
fdw_schema_name text;
time timestamp with time zone;
BEGIN
-- This will turn a local foreign table (referenced as regclass) to its fully qualified text remote table reference.
WITH a AS (SELECT ftoptions FROM pg_foreign_table WHERE ftrelid=foreign_table LIMIT 1),
b as (SELECT (pg_options_to_table(ftoptions)).* FROM a)
SELECT FORMAT('%I.%I', (SELECT option_value FROM b WHERE option_name='schema_name'), (SELECT option_value FROM b WHERE option_name='table_name'))
INTO remote_table_name;
-- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table.
SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name;
EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
RETURN time;
END
$$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid)
RETURNS TEXT AS $$
SELECT option_value FROM pg_options_to_table((
SELECT fs.srvoptions
FROM pg_foreign_table ft
LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid
WHERE ft.ftrelid = reloid
)) WHERE option_name='dbname';
$$ LANGUAGE SQL;
-- Return a set of (dbname, schema_name, table_name, updated_at)
-- It is aware of foreign tables
-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
-- Note: dbname is never quoted whereas schema and table names are when needed.
CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text)
RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz)
AS $$
WITH query_tables AS (
SELECT unnest(CDB_QueryTablesText(query)) schema_table_name
), query_tables_oid AS (
SELECT schema_table_name, schema_table_name::regclass::oid AS reloid
FROM query_tables
),
fqtn AS (
SELECT
(CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid)
ELSE current_database()
END)::text AS dbname,
quote_ident(n.nspname::text) schema_name,
quote_ident(c.relname::text) table_name,
c.relkind,
query_tables_oid.reloid
FROM query_tables_oid, pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = query_tables_oid.reloid
)
SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name,
(CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
END) AS updated_at
FROM fqtn;
$$ LANGUAGE SQL;
-- Return the last updated time of a set of tables
-- It is aware of foreign tables
-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[])
RETURNS timestamptz AS $$
WITH t AS (
SELECT unnest(tables) AS schema_table_name
), t_oid AS (
SELECT (t.schema_table_name)::regclass::oid as reloid FROM t
), t_updated_at AS (
SELECT
(CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
END) AS updated_at
FROM t_oid
LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
) SELECT max(updated_at) FROM t_updated_at;
$$ LANGUAGE SQL;

683
test/support/CDB_Overviews.sql Normal file
View File

@ -0,0 +1,683 @@
-- security definer
-- Pattern that can be used to detect overview tables and Extract
-- the intended zoom level from the table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
RETURNS TEXT
AS $$
BEGIN
RETURN '\A_vovw_(\d+)_';
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- substring(tablename from _CDB_OverviewTableDiscriminator())
-- Pattern matched by the overview tables of a given base table name.
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
RETURNS TEXT
AS $$
BEGIN
RETURN _CDB_OverviewTableDiscriminator() || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-- Name of an overview table, given the base table name and the Z level
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
RETURNS TEXT
AS $$
BEGIN
RETURN '_vovw_' || z::text || '_' || base_table;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Condition to check if a tabla is an overview table of some base table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
RETURNS BOOLEAN
AS $$
BEGIN
RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Extract the Z level from an overview table name
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
RETURNS INTEGER
AS $$
BEGIN
RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Name of the base table corresponding to an overview table
-- Scope: private.
CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
RETURNS TEXT
AS $$
BEGIN
IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
RETURN overview_table;
ELSE
RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
END IF;
END;
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Remove a dataset's existing overview tables.
-- Scope: public
-- Parameters:
-- reloid: oid of the table.
CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
RETURNS void
AS $$
DECLARE
row record;
BEGIN
FOR row IN
SELECT * FROM CDB_Overviews(reloid)
LOOP
EXECUTE Format('DROP TABLE %s;', row.overview_table);
RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table;
END LOOP;
END;
$$ LANGUAGE PLPGSQL VOLATILE;
-- Return existing overviews (if any) for a given dataset table
-- Scope: public
-- Parameters
-- reloid: oid of the input table.
-- Return relation of overviews for the table with
-- the base table oid,
-- z level of the overview and overview table oid, ordered by z.
CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
AS $$
-- FIXME: this will fail if the overview tables
-- require a explicit schema name
-- possible solutions: return table names as text instead of regclass
-- or add schema of reloid before casting to regclass
SELECT
reloid AS base_table,
_CDB_OverviewTableZ(cdb_usertables) AS z,
cdb_usertables::regclass AS overview_table
FROM CDB_UserTables()
WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
ORDER BY z;
$$ LANGUAGE SQL;
-- Return existing overviews (if any) for multiple dataset tables.
-- Scope: public
-- Parameters
-- tables: Array of input tables oids
-- Return relation of overviews for the table with
-- the base table oid,
-- z level of the overview and overview table oid, ordered by z.
-- Note: CDB_Overviews can be applied to the result of CDB_QueryTablesText
-- to obtain the overviews applicable to a query.
CREATE OR REPLACE FUNCTION CDB_Overviews(tables regclass[])
RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
AS $$
SELECT
base_table::regclass AS base_table,
_CDB_OverviewTableZ(cdb_usertables) AS z,
cdb_usertables::regclass AS overview_table
FROM
CDB_UserTables(), unnest(tables) base_table
WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
ORDER BY base_table, z;
$$ LANGUAGE SQL;
-- Schema and relation names of a table given its reloid
-- Scope: private.
-- Parameters
-- reloid: oid of the table.
-- Return (schema_name, table_name)
-- note that returned names will be quoted if necessary
CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
AS $$
BEGIN
SELECT n.nspname, c.relname
INTO STRICT schema_name, table_name
FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.oid = reloid;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Calculate the estimated extent of a cartodbfy'ed table.
-- Scope: private.
-- Parameters
-- reloid: oid of the input table.
-- Return value A box2d extent in 3857.
CREATE OR REPLACE FUNCTION _cdb_estimated_extent(reloid REGCLASS)
RETURNS box2d
AS $$
DECLARE
ext box2d;
ext_query text;
table_id record;
BEGIN
SELECT n.nspname AS schema_name, c.relname table_name INTO STRICT table_id
FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
ext_query = format(
'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
);
BEGIN
EXECUTE ext_query INTO ext;
EXCEPTION
-- This is the typical ERROR: stats for "mytable" do not exist
WHEN internal_error THEN
-- Get stats and execute again
EXECUTE format('ANALYZE %1$I', reloid);
EXECUTE ext_query INTO ext;
END;
RETURN ext;
END;
$$ LANGUAGE PLPGSQL VOLATILE;
-- Determine the max feature density of a given dataset.
-- Scope: private.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- nz: number of zoom levels to consider from z0 upward.
-- Return value: feature density (num_features / webmercator_squared_meters).
CREATE OR REPLACE FUNCTION _CDB_Feature_Density(reloid REGCLASS, nz integer)
RETURNS FLOAT8
AS $$
DECLARE
fd FLOAT8;
min_features TEXT;
n integer = 4;
c FLOAT8;
BEGIN
-- TODO: for small total count or extents we could just:
-- EXECUTE 'SELECT Count(*)/ST_Area(ST_Extent(the_geom_webmercator)) FROM ' || reloid::text || ';' INTO fd;
-- min_features is a SQL subexpression which can depend on z and represents
-- the minimum number of features to recursively consider a tile.
-- We can either use a fixed minimum number of features per tile
-- or a minimum feature density by dividing the number of features by
-- the area of tiles at level Z: c*c*power(2, -2*z)
-- with c = CDB_XYZ_Resolution(-8) (earth circumference)
min_features = '500';
SELECT CDB_XYZ_Resolution(-8) INTO c;
-- We first compute a set of *seed* tiles, of the minimum Z level, z0, such that
-- they cover the extent of the table and we have at least n of them in each
-- linear dimension (i.e. at least n*n tiles cover the extent).
-- We compute the number of features in these tiles, and recursively in
-- subtiles up to level z0 + nz. Then we compute the maximum of the feature
-- density (per tile area in webmercator squared meters) for all the
-- considered tiles.
EXECUTE Format('
WITH RECURSIVE t(x, y, z, e) AS (
WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
base AS (
SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
FROM ext
),
lim AS (
SELECT
FLOOR((ST_XMin(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x0,
FLOOR((ST_XMax(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x1,
FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMin(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y1,
FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMax(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y0
FROM ext, base
),
seed AS (
SELECT xt, yt, base.z, (
SELECT count(*) FROM %1$s
WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z)
) e
FROM base, lim, generate_series(lim.x0, lim.x1) xt, generate_series(lim.y0, lim.y1) yt
)
SELECT * from seed
UNION ALL
SELECT x*2 + xx, y*2 + yy, t.z+1, (
SELECT count(*) FROM %1$s
WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
)
FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
WHERE t.e > %2$s AND t.z < (base.z + %3$s)
)
SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
', reloid::text, min_features, nz, n, c, reloid::oid)
INTO fd;
RETURN fd;
END
$$ LANGUAGE PLPGSQL STABLE;
-- Experimental default strategy to assign a reference base Z level
-- to a cartodbfied table. The resulting Z level represents the
-- minimum scale level at which the table data can be rendered
-- without overcrowded results or loss of detail.
-- Parameters:
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- Return value: Z level as an integer
CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
RETURNS INTEGER
AS $$
DECLARE
lim FLOAT8 := 500; -- TODO: determine/parameterize this
nz integer := 4;
fd FLOAT8;
c FLOAT8;
BEGIN
-- Compute fd as an estimation of the (maximum) number
-- of features per unit of tile area (in webmercator squared meters)
SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
-- lim maximum number of (desiderable) features per tile
-- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
-- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
-- => fd*ta(z) if the average number of features per tile at level z
-- find minimum z so that fd*ta(z) <= lim
-- compute a rough 'feature density' value
SELECT CDB_XYZ_Resolution(-8) INTO c;
RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
END;
$$ LANGUAGE PLPGSQL STABLE;
-- Overview table name for a given Z level and base dataset or overview table
-- Scope: private.
-- Parameters:
-- ref reference table (can be the base table of the dataset or an existing
-- overview) from which the overview is being generated.
-- ref_z Z level of the reference table
-- overview_z Z level of the overview to be named, must be smaller than ref_z
-- Return value: the name to be used for the overview. The name is always
-- unqualified (does not include a schema name).
CREATE OR REPLACE FUNCTION _CDB_Overview_Name(ref REGCLASS, ref_z INTEGER, overview_z INTEGER)
RETURNS TEXT
AS $$
DECLARE
schema_name TEXT;
base TEXT;
suffix TEXT;
is_overview BOOLEAN;
BEGIN
SELECT * FROM _cdb_split_table_name(ref) INTO schema_name, base;
SELECT _CDB_OverviewBaseTableName(base) INTO base;
RETURN _CDB_OverviewTableName(base, overview_z);
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- Sampling reduction method.
-- Valid for any kind of geometry.
-- Scope: private.
-- reloid original table (can be the base table of the dataset or an existing
-- overview) from which the overview is being generated.
-- ref_z Z level assigned to the original table
-- overview_z Z level of the overview to be generated, must be smaller than ref_z
-- Return value: Name of the generated overview table
CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
RETURNS REGCLASS
AS $$
DECLARE
overview_rel TEXT;
fraction FLOAT8;
base_name TEXT;
class_info RECORD;
num_samples INTEGER;
BEGIN
overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
fraction := power(2, 2*(overview_z - ref_z));
-- FIXME: handle schema name for overview_rel if reloid requires it
EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
-- Estimate number of rows
SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
WHERE oid = reloid::oid;
IF class_info.relpages < 2 OR fraction > 0.5 THEN
-- We'll avoid possible CDB_RandomTids problems
EXECUTE Format('
CREATE TABLE %I AS SELECT * FROM %s WHERE random() < %s;
', overview_rel, reloid, fraction);
ELSE
num_samples := ceil(class_info.reltuples*fraction);
EXECUTE Format('
CREATE TABLE %1$I AS SELECT * FROM %2$s
WHERE ctid = ANY (
ARRAY[
(SELECT CDB_RandomTids(''%2$s'', %3$s))
]
);
', overview_rel, reloid, num_samples);
END IF;
RETURN overview_rel;
END;
$$ LANGUAGE PLPGSQL;
-- Register new overview table (post-creation chores)
-- Scope: private
-- Parameters:
-- dataset: oid of the input dataset table, It must be a cartodbfy'ed table.
-- overview_table: oid of the overview table to be registered.
-- overview_z: intended Z level for the overview table
-- This function is declared SECURITY DEFINER so it executes with the privileges
-- of the function creator to have a chance to alter the privileges of the
-- overview table to match those of the dataset. It will only perform any change
-- if the overview table belgons to the same scheme as the dataset and it
-- matches the scheme naming for overview tables.
CREATE OR REPLACE FUNCTION _CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
RETURNS VOID
AS $$
DECLARE
sql TEXT;
table_owner TEXT;
dataset_scheme TEXT;
dataset_name TEXT;
overview_scheme TEXT;
overview_name TEXT;
BEGIN
-- This function will only register a table as an overview table if it matches
-- the overviews naming scheme for the dataset and z level and the table belongs
-- to the same scheme as the the dataset
SELECT * FROM _cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name;
SELECT * FROM _cdb_split_table_name(overview_table) INTO overview_scheme, overview_name;
IF dataset_scheme = overview_scheme AND
overview_name = _CDB_OverviewTableName(dataset_name, overview_z) THEN
-- preserve the owner of the base table
SELECT u.usename
FROM pg_catalog.pg_class c JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
WHERE c.relname = dataset::text
INTO table_owner;
EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
-- preserve the table privileges
UPDATE pg_class c_to
SET relacl = c_from.relacl
FROM pg_class c_from
WHERE c_from.oid = dataset
AND c_to.oid = overview_table;
PERFORM _CDB_Add_Indexes(overview_table);
-- TODO: If metadata about existing overviews is to be stored
-- it should be done here (CDB_Overviews would consume such metadata)
END IF;
END
$$ LANGUAGE PLPGSQL SECURITY DEFINER;
-- Dataset attributes (column names other than the
-- CartoDB primary key and geometry columns) which should be aggregated
-- in aggregated overviews.
-- Scope: private.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- Return value: set of attribute names
CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes(reloid REGCLASS)
RETURNS SETOF information_schema.sql_identifier
AS $$
SELECT c FROM CDB_ColumnNames(reloid) c, _CDB_Columns() cdb
WHERE c NOT IN (
cdb.pkey, cdb.geomcol, cdb.mercgeomcol
)
$$ LANGUAGE SQL STABLE;
-- List of dataset attributes to be aggregated in aggregated overview
-- as a comma-separated SQL expression.
-- Scope: private.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- Return value: SQL subexpression as text
CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes_Expression(reloid REGCLASS)
RETURNS TEXT
AS $$
DECLARE
attr_list TEXT;
BEGIN
SELECT string_agg(s.c, ',') FROM (
SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
) AS s INTO attr_list;
RETURN attr_list;
END
$$ LANGUAGE PLPGSQL STABLE;
-- SQL Aggregation expression for a datase attribute
-- Scope: private.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- column_name: column to be aggregated
-- table_alias: (optional) table qualifier for the column to be aggregated
-- Return SQL subexpression as text with aggregated attribute aliased
-- with its original name.
CREATE OR REPLACE FUNCTION _CDB_Attribute_Aggregation_Expression(reloid REGCLASS, column_name TEXT, table_alias TEXT DEFAULT '')
RETURNS TEXT
AS $$
DECLARE
column_type TEXT;
qualified_column TEXT;
BEGIN
IF table_alias <> '' THEN
qualified_column := Format('%I.%I', table_alias, column_name);
ELSE
qualified_column := Format('%I', column_name);
END IF;
column_type := CDB_ColumnType(reloid, column_name);
CASE column_type
WHEN 'double precision', 'real', 'integer', 'bigint' THEN
RETURN Format('AVG(%s)::' || column_type, qualified_column);
WHEN 'text' THEN
-- TODO: we could define a new aggregate function that returns distinct
-- separated values with a limit, adding ellipsis if more values existed
-- e.g. with '/' as separator and a limit of three:
-- 'A', 'B', 'A', 'C', 'D' => 'A/B/C/...'
-- Other ideas: if value is unique then use it, otherwise use something
-- like '*' or '(varies)' or '(multiple values)', or NULL
-- Using 'string_agg(' || qualified_column || ',''/'')'
-- here causes
RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
ELSE
RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
END CASE;
END
$$ LANGUAGE PLPGSQL IMMUTABLE;
-- List of dataset aggregated attributes as a comma-separated SQL expression.
-- Scope: private.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- table_alias: (optional) table qualifier for the columns to be aggregated
-- Return value: SQL subexpression as text
CREATE OR REPLACE FUNCTION _CDB_Aggregated_Attributes_Expression(reloid REGCLASS, table_alias TEXT DEFAULT '')
RETURNS TEXT
AS $$
DECLARE
attr_list TEXT;
BEGIN
SELECT string_agg(_CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
FROM (
SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
) AS s INTO attr_list;
RETURN attr_list;
END
$$ LANGUAGE PLPGSQL STABLE;
-- Array of geometry types detected in a cartodbfied table
-- For effciency only look at a limited number of rwos.
-- Parameters
-- reloid: oid of the input table. It must be a cartodbfy'ed table.
-- Return value: array of geometry type names
CREATE OR REPLACE FUNCTION _CDB_GeometryTypes(reloid REGCLASS)
RETURNS TEXT[]
AS $$
DECLARE
gtypes TEXT[];
BEGIN
EXECUTE Format('
SELECT array_agg(DISTINCT ST_GeometryType(the_geom)) FROM (
SELECT the_geom FROM %s
WHERE (the_geom is not null) LIMIT 10
) as geom_types
', reloid)
INTO gtypes;
RETURN gtypes;
END
$$ LANGUAGE PLPGSQL STABLE;
-- Experimental Overview reduction method for point datasets.
-- It clusters the points using a grid, then aggregates the point in each
-- cluster into a point at the centroid of the clustered records.
-- Scope: private.
-- Parameters:
-- reloid original table (can be the base table of the dataset or an existing
-- overview) from which the overview is being generated.
-- ref_z Z level assigned to the original table
-- overview_z Z level of the overview to be generated, must be smaller than ref_z
-- Return value: Name of the generated overview table
CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
RETURNS REGCLASS
AS $$
DECLARE
overview_rel TEXT;
reduction FLOAT8;
base_name TEXT;
grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
grid_m FLOAT8;
aggr_attributes TEXT;
attributes TEXT;
columns TEXT;
gtypes TEXT[];
BEGIN
SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
-- This strategy only supports datasets with point geomety
RETURN NULL;
RETURN 'x';
END IF;
--TODO: check applicability: geometry type, minimum number of points...
overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
-- compute grid cell size using the overview_z dimension...
SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
attributes := _CDB_Aggregable_Attributes_Expression(reloid);
aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
IF attributes <> '' THEN
attributes := ', ' || attributes;
END IF;
IF aggr_attributes <> '' THEN
aggr_attributes := aggr_attributes || ', ';
END IF;
-- compute the resulting columns in the same order as in the base table
-- cartodb_id,
-- ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom,
-- ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator
-- %4$s
WITH cols AS (
SELECT
CASE c
WHEN 'cartodb_id' THEN 'cartodb_id'
WHEN 'the_geom' THEN
'ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom'
WHEN 'the_geom_webmercator' THEN
'ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator'
ELSE c
END AS column
FROM CDB_ColumnNames(reloid) c
)
SELECT string_agg(s.column, ',') FROM (
SELECT * FROM cols
) AS s INTO columns;
-- FIXME: handle schema name for overview_rel if reloid requires it
EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
-- Now we cluster the data using a grid of size grid_m
-- and selecte the centroid (average coordinates) of each cluster.
-- If we had a selected numeric attribute of interest we could use it
-- as a weight for the average coordinates.
EXECUTE Format('
CREATE TABLE %3$I AS
WITH clusters AS (
SELECT
%5$s
count(*) AS n,
SUM(ST_X(f.the_geom_webmercator)) AS sx,
SUM(ST_Y(f.the_geom_webmercator)) AS sy,
Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
MIN(cartodb_id) AS cartodb_id
FROM %1$s f
GROUP BY gx, gy
)
SELECT %6$s FROM clusters
', reloid::text, grid_m, overview_rel, attributes, aggr_attributes, columns);
RETURN overview_rel;
END;
$$ LANGUAGE PLPGSQL;
-- Create overview tables for a dataset.
-- Scope: public
-- Parameters:
-- reloid: oid of the input table. It must be a cartodbfy'ed table with
-- vector features.
-- refscale_strategy: function that computes the reference Z of the dataset
-- reduce_strategy: function that generates overviews from a base table
-- or higher level overview. The overview tables
-- created by the strategy must have the same columns
-- as the base table and in the same order.
-- Return value: Array with the names of the generated overview tables
CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
RETURNS text[]
AS $$
DECLARE
ref_z integer;
overviews_z integer[];
base_z integer;
base_rel REGCLASS;
overview_z integer;
overview_tables REGCLASS[];
overviews_step integer := 1;
BEGIN
-- Determine the referece zoom level
EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
-- Determine overlay zoom levels
-- TODO: should be handled by the refscale_strategy?
overview_z := ref_z - 1;
WHILE overview_z >= 0 LOOP
SELECT array_append(overviews_z, overview_z) INTO overviews_z;
overview_z := overview_z - overviews_step;
END LOOP;
-- Create overlay tables
base_z := ref_z;
base_rel := reloid;
FOREACH overview_z IN ARRAY overviews_z LOOP
EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
IF base_rel IS NULL THEN
EXIT;
END IF;
base_z := overview_z;
PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
SELECT array_append(overview_tables, base_rel) INTO overview_tables;
END LOOP;
RETURN overview_tables;
END;
$$ LANGUAGE PLPGSQL;

143
test/support/CDB_TableMetadata.sql Normal file
View File

@ -0,0 +1,143 @@
CREATE TABLE IF NOT EXISTS
public.CDB_TableMetadata (
tabname regclass not null primary key,
updated_at timestamp with time zone not null default now()
);
CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS
SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at
FROM public.CDB_TableMetadata, pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid;
-- No one can see this
-- Updates are only possible trough the security definer trigger
-- GRANT SELECT ON public.CDB_TableMetadata TO public;
--
-- Trigger logging updated_at in the CDB_TableMetadata
-- and notifying cdb_tabledata_update with table name as payload.
--
-- Attach to tables like this:
--
-- CREATE trigger track_updates
-- AFTER INSERT OR UPDATE OR TRUNCATE OR DELETE ON <tablename>
-- FOR EACH STATEMENT
-- EXECUTE PROCEDURE cdb_tablemetadata_trigger();
--
-- NOTE: _never_ attach to CDB_TableMetadata ...
--
CREATE OR REPLACE FUNCTION CDB_TableMetadata_Trigger()
RETURNS trigger AS
$$
BEGIN
-- Guard against infinite loop
IF TG_RELID = 'public.CDB_TableMetadata'::regclass::oid THEN
RETURN NULL;
END IF;
-- Cleanup stale entries
DELETE FROM public.CDB_TableMetadata
WHERE NOT EXISTS (
SELECT oid FROM pg_class WHERE oid = tabname
);
WITH nv as (
SELECT TG_RELID as tabname, NOW() as t
), updated as (
UPDATE public.CDB_TableMetadata x SET updated_at = nv.t
FROM nv WHERE x.tabname = nv.tabname
RETURNING x.tabname
)
INSERT INTO public.CDB_TableMetadata SELECT nv.*
FROM nv LEFT JOIN updated USING(tabname)
WHERE updated.tabname IS NULL;
RETURN NULL;
END;
$$
LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
--
-- Trigger invalidating varnish whenever CDB_TableMetadata
-- record change.
--
CREATE OR REPLACE FUNCTION _CDB_TableMetadata_Updated()
RETURNS trigger AS
$$
DECLARE
tabname TEXT;
rec RECORD;
found BOOL;
BEGIN
IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
tabname = NEW.tabname;
ELSE
tabname = OLD.tabname;
END IF;
-- Notify table data update
-- This needs a little bit more of research regarding security issues
-- see https://github.com/CartoDB/cartodb/pull/241
-- PERFORM pg_notify('cdb_tabledata_update', tabname);
--RAISE NOTICE 'Table % was updated', tabname;
-- This will be needed until we'll have someone listening
-- on the event we just broadcasted:
--
-- LISTEN cdb_tabledata_update;
--
-- Call the first varnish invalidation function owned
-- by a superuser found in cartodb or public schema
-- (in that order)
found := false;
FOR rec IN SELECT u.usesuper, u.usename, n.nspname, p.proname
FROM pg_proc p, pg_namespace n, pg_user u
WHERE p.proname = 'cdb_invalidate_varnish'
AND p.pronamespace = n.oid
AND n.nspname IN ('public', 'cartodb')
AND u.usesysid = p.proowner
AND u.usesuper
ORDER BY n.nspname
LOOP
EXECUTE 'SELECT ' || quote_ident(rec.nspname) || '.'
|| quote_ident(rec.proname)
|| '(' || quote_literal(tabname) || ')';
found := true;
EXIT;
END LOOP;
IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF;
RETURN NULL;
END;
$$
LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
DROP TRIGGER IF EXISTS table_modified ON CDB_TableMetadata;
-- NOTE: on DELETE we would be unable to convert the table
-- oid (regclass) to its name
CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
_CDB_TableMetadata_Updated();
-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata
CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass)
RETURNS void AS
$$
BEGIN
WITH upsert AS (
UPDATE public.cdb_tablemetadata
SET updated_at = NOW()
WHERE tabname = tablename
RETURNING *
)
INSERT INTO public.cdb_tablemetadata (tabname, updated_at)
SELECT tablename, NOW()
WHERE NOT EXISTS (SELECT * FROM upsert);
END;
$$
LANGUAGE 'plpgsql' VOLATILE STRICT;

28
test/support/CDB_UserTables.sql Normal file
View File

@ -0,0 +1,28 @@
-- Function returning list of cartodb user tables
--
-- The optional argument restricts the result to tables
-- of the specified access type.
--
-- Currently accepted permissions are: 'public', 'private' or 'all'
--
DROP FUNCTION IF EXISTS CDB_UserTables(text);
CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
RETURNS SETOF name
AS $$
SELECT c.relname
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')
ELSE false END;
$$ LANGUAGE 'sql';
-- This is to migrate from pre-0.2.0 version
-- See http://github.com/CartoDB/cartodb-postgresql/issues/36
GRANT EXECUTE ON FUNCTION CDB_UserTables(text) TO public;

30
test/support/CDB_ZoomFromScale.sql Normal file
View File

@ -0,0 +1,30 @@
CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
BEGIN
CASE
WHEN scaleDenominator > 500000000 THEN RETURN 0;
WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
WHEN scaleDenominator <= 100 THEN RETURN 23;
END CASE;
END
$$ LANGUAGE plpgsql IMMUTABLE;