diff --git a/app/app.js b/app/app.js index 4f87a601..1f44b399 100644 --- a/app/app.js +++ b/app/app.js @@ -29,7 +29,6 @@ var UserIndexer = require('../batch/user_indexer'); var JobBackend = require('../batch/job_backend'); var JobCanceller = require('../batch/job_canceller'); var UserDatabaseMetadataService = require('../batch/user_database_metadata_service'); -var QueryTablesApi = require('./services/query-tables-api'); var cors = require('./middlewares/cors'); @@ -71,7 +70,6 @@ function App() { // consider entries expired after these many milliseconds (10 minutes by default) maxAge: global.settings.tableCacheMaxAge || 1000*60*10 }); - var queryTablesApi = new QueryTablesApi(tableCache); // Size based on https://github.com/CartoDB/cartodb.js/blob/3.15.2/src/geo/layer_definition.js#L72 var SQL_QUERY_BODY_LOG_MAX_LENGTH = 2000; @@ -190,7 +188,7 @@ function App() { var genericController = new GenericController(); genericController.route(app); - var queryController = new QueryController(userDatabaseService, queryTablesApi, statsd_client); + var queryController = new QueryController(userDatabaseService, statsd_client); queryController.route(app); var jobController = new JobController(userDatabaseService, jobBackend, jobCanceller); diff --git a/app/controllers/query_controller.js b/app/controllers/query_controller.js index 0ca2f819..4fd0bd0a 100644 --- a/app/controllers/query_controller.js +++ b/app/controllers/query_controller.js @@ -4,23 +4,22 @@ var _ = require('underscore'); var step = require('step'); var assert = require('assert'); var PSQL = require('cartodb-psql'); - +var QueryTables = require('node-cartodb-query-tables'); var AuthApi = require('../auth/auth_api'); +var queryMayWrite = require('../utils/query_may_write'); var CdbRequest = require('../models/cartodb_request'); var formats = require('../models/formats'); var sanitize_filename = require('../utils/filename_sanitizer'); var getContentDisposition = require('../utils/content_disposition'); -var generateCacheKey = require('../utils/cache_key_generator'); var handleException = require('../utils/error_handler'); var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default var cdbReq = new CdbRequest(); -function QueryController(userDatabaseService, queryTablesApi, statsd_client) { - this.queryTablesApi = queryTablesApi; +function QueryController(userDatabaseService, statsd_client) { this.statsd_client = statsd_client; this.userDatabaseService = userDatabaseService; } @@ -133,22 +132,22 @@ QueryController.prototype.handleQuery = function (req, res) { checkAborted('queryExplain'); - self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(authDbParams, sql, this); + var pg = new PSQL(authDbParams, {}, { destroyOnError: true }); + QueryTables.getAffectedTablesFromQuery(pg, sql, this); }, - function setHeaders(err, queryExplainResult) { + function setHeaders(err, affectedTables) { assert.ifError(err); + var mayWrite = queryMayWrite(sql); if ( req.profiler ) { req.profiler.done('queryExplain'); } checkAborted('setHeaders'); - if (!dbopts.authenticated) { - var affected_tables = queryExplainResult.affectedTables; - for ( var i = 0; i < affected_tables.length; ++i ) { - var t = affected_tables[i]; - if ( t.match(/\bpg_/) ) { + for ( var i = 0; i < affectedTables.tables.length; ++i ) { + var t = affectedTables.tables[i]; + if ( t.table_name.match(/\bpg_/) ) { var e = new SyntaxError("system tables are forbidden"); e.http_status = 403; throw(e); @@ -171,16 +170,17 @@ QueryController.prototype.handleQuery = function (req, res) { if (cachePolicy === 'persist') { res.header('Cache-Control', 'public,max-age=' + ONE_YEAR_IN_SECONDS); } else { - var maxAge = (queryExplainResult.mayWrite) ? 0 : ONE_YEAR_IN_SECONDS; + var maxAge = (mayWrite) ? 0 : ONE_YEAR_IN_SECONDS; res.header('Cache-Control', 'no-cache,max-age='+maxAge+',must-revalidate,public'); } // Only set an X-Cache-Channel for responses we want Varnish to cache. - if (queryExplainResult.affectedTables.length > 0 && !queryExplainResult.mayWrite) { - res.header('X-Cache-Channel', generateCacheKey(dbopts.dbname, queryExplainResult.affectedTables)); + if (affectedTables.tables.length > 0 && !mayWrite) { + res.header('X-Cache-Channel', affectedTables.getCacheChannel()); + res.header('Surrogate-Key', affectedTables.key()); } - res.header('Last-Modified', new Date(queryExplainResult.lastModified).toUTCString()); + res.header('Last-Modified', new Date(affectedTables.getLastUpdatedAt()).toUTCString()); return null; }, diff --git a/app/services/query-tables-api.js b/app/services/query-tables-api.js deleted file mode 100644 index 904bc43f..00000000 --- a/app/services/query-tables-api.js +++ /dev/null @@ -1,112 +0,0 @@ -var PSQL = require('cartodb-psql'); - -var generateMD5 = require('../utils/md5'); -var queryMayWrite = require('../utils/query_may_write'); - -function QueryTablesApi(tableCache) { - this.tableCache = tableCache; -} - -module.exports = QueryTablesApi; - -QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (connectionParams, sql, callback) { - var self = this; - - var cacheKey = sqlCacheKey(connectionParams.user, sql); - var queryExplainResult = this.tableCache.get(cacheKey); - - if (queryExplainResult) { - queryExplainResult.hits++; - getLastUpdatedTime(connectionParams, queryExplainResult.affectedTables, function(err, lastUpdatedTime) { - return callback(null, { - affectedTables: queryExplainResult.affectedTables, - lastModified: lastUpdatedTime, - mayWrite: queryExplainResult.mayWrite - }); - }); - } else { - getAffectedTablesAndLastUpdatedTime(connectionParams, sql, function(err, affectedTablesAndLastUpdatedTime) { - var queryExplainResult = { - affectedTables: affectedTablesAndLastUpdatedTime.affectedTables, - mayWrite: queryMayWrite(sql), - hits: 1 - }; - - self.tableCache.set(cacheKey, queryExplainResult); - - return callback(null, { - affectedTables: queryExplainResult.affectedTables, - lastModified: affectedTablesAndLastUpdatedTime.lastUpdatedTime, - mayWrite: queryExplainResult.mayWrite - }); - }); - } -}; - -function getAffectedTablesAndLastUpdatedTime(connectionParams, sql, callback) { - var query = [ - 'WITH querytables AS (', - 'SELECT * FROM CDB_QueryTablesText($quotesql$' + sql + '$quotesql$) as tablenames', - ')', - 'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max', - 'FROM CDB_TableMetadata m', - 'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])' - ].join(' '); - - var pg = new PSQL(connectionParams, {}, { destroyOnError: true }); - - pg.query(query, function handleAffectedTablesAndLastUpdatedTimeRows(err, resultSet) { - resultSet = resultSet || {}; - var rows = resultSet.rows || []; - - logIfError(err, sql, rows); - - var result = rows[0] || {}; - - // This is an Array, so no need to split into parts - var tableNames = result.tablenames || []; - var lastUpdatedTime = (Number.isFinite(result.max)) ? (result.max * 1000) : Date.now(); - - return callback(null, { - affectedTables: tableNames, - lastUpdatedTime: lastUpdatedTime - }); - }, true); -} - -function getLastUpdatedTime(connectionParams, tableNames, callback) { - if (!Array.isArray(tableNames) || tableNames.length === 0) { - return callback(null, Date.now()); - } - - var query = [ - 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max', - 'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[', - tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','), - '])' - ].join(' '); - - var pg = new PSQL(connectionParams, {}, { destroyOnError: true }); - - pg.query(query, function handleLastUpdatedTimeRows (err, resultSet) { - resultSet = resultSet || {}; - var rows = resultSet.rows || []; - - var result = rows[0] || {}; - - var lastUpdatedTime = (Number.isFinite(result.max)) ? (result.max * 1000) : Date.now(); - - return callback(null, lastUpdatedTime); - }, true); -} - -function logIfError(err, sql, rows) { - if (err || rows.length !== 1) { - var errorMessage = (err && err.message) || 'unknown error'; - console.error("Error on query explain '%s': %s", sql, errorMessage); - } -} - -function sqlCacheKey(user, sql) { - return user + ':' + generateMD5(sql); -} diff --git a/package.json b/package.json index 6f921236..6ce5abfb 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "step-profiler": "~0.1.0", "topojson": "0.0.8", "underscore": "~1.6.0", - "queue-async": "~1.0.7" + "queue-async": "~1.0.7", + "node-cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" }, "devDependencies": { "istanbul": "~0.4.2", diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index cdeefa2e..686e5aa9 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -789,7 +789,7 @@ it('DROP TABLE with GET and auth', function(done){ }); }); -test('CREATE FUNCTION with GET and auth', function(done){ +it('CREATE FUNCTION with GET and auth', function(done){ assert.response(app, { url: "/api/v1/sql?" + querystring.stringify({ q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'', diff --git a/test/prepare_db.sh b/test/prepare_db.sh index 92a17fa0..7563885f 100755 --- a/test/prepare_db.sh +++ b/test/prepare_db.sh @@ -85,10 +85,12 @@ if test x"$PREPARE_PGSQL" = xyes; then # TODO: send in a single run, togheter with test.sql psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o support/CDB_QueryStatements.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o support/CDB_QueryTables.sql - psql -f support/CDB_QueryStatements.sql ${TEST_DB} - psql -f support/CDB_QueryTables.sql ${TEST_DB} + for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews + do + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o support/$i.sql + cat support/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \ + | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + done fi diff --git a/test/run_tests.sh b/test/run_tests.sh index c00c3c2a..8741199c 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -135,7 +135,7 @@ echo if test x"$OPT_COVERAGE" = xyes; then echo "Running tests with coverage" - ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd --trace -t 5000 ${TESTS} else echo "Running tests" mocha -u tdd -t 5000 ${TESTS} diff --git a/test/support/CDB_CartodbfyTable.sql b/test/support/CDB_CartodbfyTable.sql new file mode 100644 index 00000000..97006969 --- /dev/null +++ b/test/support/CDB_CartodbfyTable.sql @@ -0,0 +1,1317 @@ +-- Depends on: +-- * CDB_Helper.sql +-- * CDB_ExtensionUtils.sql +-- * CDB_TransformToWebmercator.sql +-- * CDB_TableMetadata.sql +-- * CDB_Quota.sql +-- * _CDB_UserQuotaInBytes() function, installed by rails +-- (user.rebuild_quota_trigger, called by rake task cartodb:db:update_test_quota_trigger) + +-- 1) Required checks before running cartodbfication +-- Either will pass silenty or raise an exception +CREATE OR REPLACE FUNCTION _CDB_check_prerequisites(schema_name TEXT, reloid REGCLASS) +RETURNS void +AS $$ +DECLARE + sql TEXT; +BEGIN + IF cartodb.schema_exists(schema_name) = false THEN + RAISE EXCEPTION 'Invalid schema name "%"', schema_name; + END IF; + + -- TODO: Check that user quota is set ? + BEGIN + EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name::text) INTO sql; + EXCEPTION WHEN undefined_function THEN + RAISE EXCEPTION 'Please set user quota before cartodbfying tables.'; + END; +END; +$$ LANGUAGE PLPGSQL; + +-- Drop cartodb triggers (might prevent changing columns) +CREATE OR REPLACE FUNCTION _CDB_drop_triggers(reloid REGCLASS) + RETURNS void +AS $$ +DECLARE + sql TEXT; +BEGIN + -- "track_updates" + sql := Format('DROP TRIGGER IF EXISTS track_updates ON %s', reloid::text); + EXECUTE sql; + + -- "update_the_geom_webmercator" + sql := Format('DROP TRIGGER IF EXISTS update_the_geom_webmercator_trigger ON %s', reloid::text); + EXECUTE sql; + + -- "test_quota" and "test_quota_per_row" + sql := Format('DROP TRIGGER IF EXISTS test_quota ON %s', reloid::text); + EXECUTE sql; + sql := Format('DROP TRIGGER IF EXISTS test_quota_per_row ON %s', reloid::text); + EXECUTE sql; +END; +$$ LANGUAGE PLPGSQL; + + +-- Cartodb_id creation & validation or renaming if invalid +CREATE OR REPLACE FUNCTION _CDB_create_cartodb_id_column(reloid REGCLASS) + RETURNS void +AS $$ +DECLARE + sql TEXT; + rec RECORD; + rec2 RECORD; + had_column BOOLEAN; + i INTEGER; + new_name TEXT; + cartodb_id_name TEXT; +BEGIN + << cartodb_id_setup >> + LOOP --{ + had_column := FALSE; + BEGIN + sql := Format('ALTER TABLE %s ADD cartodb_id SERIAL NOT NULL UNIQUE', reloid::text); + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + cartodb_id_name := 'cartodb_id'; + EXIT cartodb_id_setup; + EXCEPTION + WHEN duplicate_column THEN + RAISE NOTICE 'Column cartodb_id already exists'; + had_column := TRUE; + WHEN others THEN + RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE; + END; + + IF had_column THEN + SELECT pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') + AS seq INTO rec2; + + -- Check data type is an integer + SELECT + pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') as seq, + t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a + WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = 'cartodb_id' + INTO STRICT rec; + + -- 20=int2, 21=int4, 23=int8 + IF rec.oid NOT IN (20,21,23) THEN -- { + RAISE NOTICE 'Existing cartodb_id field is of invalid type % (need int2, int4 or int8), renaming', rec.typname; + ELSIF rec.seq IS NULL THEN -- }{ + RAISE NOTICE 'Existing cartodb_id field does not have an associated sequence, renaming'; + ELSE -- }{ + sql := Format('ALTER TABLE %s ALTER COLUMN cartodb_id SET NOT NULL', reloid::text); + IF NOT EXISTS ( SELECT c.conname FROM pg_constraint c, pg_attribute a + WHERE c.conkey = ARRAY[a.attnum] AND c.conrelid = reloid + AND a.attrelid = reloid + AND NOT a.attisdropped + AND a.attname = 'cartodb_id' + AND c.contype IN ( 'u', 'p' ) ) -- unique or pkey + THEN + sql := sql || ', ADD unique(cartodb_id)'; + END IF; + BEGIN + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + cartodb_id_name := 'cartodb_id'; + EXIT cartodb_id_setup; + EXCEPTION + WHEN unique_violation OR not_null_violation THEN + RAISE NOTICE '%, renaming', SQLERRM; + WHEN others THEN + RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE; + END; + END IF; -- } + + -- invalid column, need rename and re-create it + i := 0; + << rename_column >> + LOOP --{ + new_name := '_cartodb_id' || i; + BEGIN + sql := Format('ALTER TABLE %s RENAME COLUMN cartodb_id TO %I', reloid::text, new_name); + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + EXCEPTION + WHEN duplicate_column THEN + i := i+1; + CONTINUE rename_column; + WHEN others THEN + RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE; + END; + cartodb_id_name := new_name; + EXIT rename_column; + END LOOP; --} + CONTINUE cartodb_id_setup; + END IF; + END LOOP; -- } + + -- Try to copy data from new name if possible + IF new_name IS NOT NULL THEN + RAISE NOTICE 'Trying to recover data from % column', new_name; + BEGIN + -- Copy existing values to new field + -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows + -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost + sql := Format('ALTER TABLE %s ALTER cartodb_id TYPE int USING %I::integer', reloid::text, new_name); + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + + -- Find max value + sql := Format('SELECT coalesce(max(cartodb_id), 0) as max FROM %s', reloid::text); + RAISE DEBUG 'Running %', sql; + EXECUTE sql INTO rec; + + -- Find sequence name + SELECT pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') + AS seq INTO rec2; + + -- Reset sequence name + sql := Format('ALTER SEQUENCE %s RESTART WITH %s', rec2.seq::text, rec.max + 1); + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + + -- Drop old column (all went fine if we got here) + sql := Format('ALTER TABLE %s DROP %I', reloid::text, new_name); + RAISE DEBUG 'Running %', sql; + EXECUTE sql; + + EXCEPTION + WHEN others THEN + RAISE NOTICE 'Could not initialize cartodb_id with existing values: % (%)', + SQLERRM, SQLSTATE; + END; + END IF; + + -- Set primary key of the table if not already present (e.g. tables created from SQL API) + IF cartodb_id_name IS NULL THEN + RAISE EXCEPTION 'Cartodbfying % (Didnt get cartodb_id field name)', reloid; + END IF; + BEGIN + sql := Format('ALTER TABLE %s ADD PRIMARY KEY (cartodb_id)', reloid::text); + EXECUTE sql; + EXCEPTION + WHEN others THEN + RAISE DEBUG 'Table % Already had PRIMARY KEY', reloid; + END; + +END; +$$ LANGUAGE PLPGSQL; + + +-- Create all triggers +-- NOTE: drop/create has the side-effect of re-enabling disabled triggers +CREATE OR REPLACE FUNCTION _CDB_create_triggers(schema_name TEXT, reloid REGCLASS) +RETURNS void +AS $$ +DECLARE + sql TEXT; +BEGIN +-- "track_updates" + sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON ' + || reloid::text + || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()'; + EXECUTE sql; + +-- "update_the_geom_webmercator" +-- TODO: why _before_ and not after ? + sql := 'CREATE trigger update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON ' + || reloid::text + || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()'; + EXECUTE sql; + +-- "test_quota" and "test_quota_per_row" + + sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON ' + || reloid::text + || ' EXECUTE PROCEDURE public.CDB_CheckQuota(0.1, ''-1'', ''' + || schema_name::text + || ''')'; + EXECUTE sql; + + sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON ' + || reloid::text + || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', ''' + || schema_name::text + || ''')'; + EXECUTE sql; +END; +$$ LANGUAGE PLPGSQL; + +-- 8.b) Create all raster triggers +-- NOTE: drop/create has the side-effect of re-enabling disabled triggers +CREATE OR REPLACE FUNCTION _CDB_create_raster_triggers(schema_name TEXT, reloid REGCLASS) + RETURNS void +AS $$ +DECLARE + sql TEXT; +BEGIN +-- "track_updates" + sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON ' + || reloid::text + || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()'; + EXECUTE sql; + +-- "test_quota" and "test_quota_per_row" + + sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON ' + || reloid::text + || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', ''' + || schema_name::text + || ''')'; + EXECUTE sql; + + sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON ' + || reloid::text + || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', ''' + || schema_name::text + || ''')'; + EXECUTE sql; +END; +$$ LANGUAGE PLPGSQL; + + + +-- Update the_geom_webmercator +CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator() + RETURNS trigger +AS $$ +BEGIN + NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom); + RETURN NEW; +END; +$$ LANGUAGE plpgsql VOLATILE; + +--- Trigger to update the updated_at column. No longer added by default +--- but kept here for compatibility with old tables which still have this behavior +--- and have it added +CREATE OR REPLACE FUNCTION _CDB_update_updated_at() + RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql VOLATILE; + +-- Auxiliary function +CREATE OR REPLACE FUNCTION cartodb._CDB_is_raster_table(schema_name TEXT, reloid REGCLASS) + RETURNS BOOLEAN +AS $$ +DECLARE + sql TEXT; + is_raster BOOLEAN; + rel_name TEXT; +BEGIN + IF cartodb.schema_exists(schema_name) = FALSE THEN + RAISE EXCEPTION 'Invalid schema name "%"', schema_name; + END IF; + + SELECT relname FROM pg_class WHERE oid=reloid INTO rel_name; + + BEGIN + sql := 'SELECT the_raster_webmercator FROM ' + || quote_ident(schema_name::TEXT) + || '.' + || quote_ident(rel_name::TEXT) + || ' LIMIT 1'; + is_raster = TRUE; + EXECUTE sql; + + EXCEPTION WHEN undefined_column THEN + is_raster = FALSE; + END; + + RETURN is_raster; +END; +$$ LANGUAGE PLPGSQL; + + + +-- //////////////////////////////////////////////////// + +-- Ensure a table is a "cartodb" table (See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table) + +DROP FUNCTION IF EXISTS CDB_CartodbfyTable(reloid REGCLASS); +CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(reloid REGCLASS) +RETURNS REGCLASS +AS $$ +BEGIN + RETURN cartodb.CDB_CartodbfyTable('public', reloid); +END; +$$ LANGUAGE PLPGSQL; + + +-- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +-- +-- NEW CARTODBFY CODE FROM HERE ON DOWN +-- +-- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +-- +-- CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS) +-- +-- Main function, calls the following functions, with a little +-- logic before the table re-write to avoid re-writing if the table +-- already has all the necessary columns in place. +-- +-- It returns the destoid of the table. If no rewritting is needed +-- the return value will be equal to reloid. +-- +-- +-- (0) _CDB_check_prerequisites +-- As before, this checks the prerequisites before trying to cartodbfy +-- +-- (1) _CDB_drop_triggers +-- As before, this drops all the metadata and geom sync triggers +-- +-- (2) _CDB_Has_Usable_Primary_ID() +-- Returns TRUE if it can find a unique and not null integer primary key named +-- 'cartodb_id' or can rename an existing key. +-- Returns FALSE otherwise. +-- +-- (3) _CDB_Has_Usable_Geom() +-- Looks for existing EPSG:4326 and EPSG:3857 geometry columns, and +-- renames them to the standard names if it can find them, returning TRUE. +-- If it cannot find both columns in the right EPSG, returns FALSE. +-- +-- (4) _CDB_Rewrite_Table() +-- If table does not have a usable primary key and both usable geom +-- columns it needs to be re-written. Function constructs an appropriate +-- CREATE TABLE AS SELECT... query and executes it. +-- +-- (5) _CDB_Add_Indexes() +-- Checks the primary key column for primary key constraint, adds it if +-- missing. Check geometry columns for GIST indexes and adds them if missing. +-- +-- (6) _CDB_create_triggers() +-- Adds the system metadata and geometry column update triggers back +-- onto the table. +-- +-- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + + +CREATE OR REPLACE FUNCTION _CDB_Columns(OUT pkey TEXT, OUT geomcol TEXT, OUT mercgeomcol TEXT) +RETURNS record +AS $$ +BEGIN + +pkey := 'cartodb_id'; +geomcol := 'the_geom'; +mercgeomcol := 'the_geom_webmercator'; + +END; +$$ LANGUAGE 'plpgsql'; + + +CREATE OR REPLACE FUNCTION _CDB_Error(message TEXT, funcname TEXT DEFAULT '_CDB_Error') +RETURNS void +AS $$ +BEGIN + + RAISE EXCEPTION 'CDB(%): %', funcname, message; + +END; +$$ LANGUAGE 'plpgsql'; + + +CREATE OR REPLACE FUNCTION _CDB_SQL(sql TEXT, funcname TEXT DEFAULT '_CDB_SQL') +RETURNS void +AS $$ +BEGIN + + RAISE DEBUG 'CDB(%): %', funcname, sql; + EXECUTE sql; + + EXCEPTION + WHEN others THEN + RAISE EXCEPTION 'CDB(%:%:%): %', funcname, SQLSTATE, SQLERRM, sql; + +END; +$$ LANGUAGE 'plpgsql'; + + +-- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe and length +-- aware. Find a unique relation name in the given schema, starting from the +-- template given. If the template is already unique, just return it; +-- otherwise, append an increasing integer until you find a unique variant. +CREATE OR REPLACE FUNCTION _CDB_Unique_Relation_Name(schemaname TEXT, relationname TEXT) +RETURNS TEXT +AS $$ +DECLARE + rec RECORD; + i INTEGER; + newrelname TEXT; +BEGIN + + RAISE EXCEPTION '_CDB_Unique_Relation_Name is DEPRECATED. Use _CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; + +END; +$$ LANGUAGE 'plpgsql'; + + +-- DEPRECATED: Use _CDB_Unique_Column_Identifier since it's UTF8 Safe and length +-- aware. Find a unique column name in the given relation, starting from the +-- column name given. If the column name is already unique, just return it; +-- otherwise, append an increasing integer until you find a unique variant. +CREATE OR REPLACE FUNCTION _CDB_Unique_Column_Name(reloid REGCLASS, columnname TEXT) +RETURNS TEXT +AS $$ +DECLARE + rec RECORD; + i INTEGER; + newcolname TEXT; +BEGIN + + RAISE EXCEPTION '_CDB_Unique_Column_Name is DEPRECATED. Use _CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; + +END; +$$ LANGUAGE 'plpgsql'; + + +-- Find out if the table already has a usable primary key +-- If the table has both a usable key and usable geometry +-- we can no-op on the table copy and just ensure that the +-- indexes and triggers are in place +CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Primary_ID(reloid REGCLASS) + RETURNS BOOLEAN +AS $$ +DECLARE + rec RECORD; + const RECORD; + i INTEGER; + sql TEXT; + useable_key BOOLEAN = false; +BEGIN + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 'entered function'; + + -- Read in the names of the CartoDB columns + const := _CDB_Columns(); + + -- Do we already have a properly named column? + SELECT a.attname, i.indisprimary, i.indisunique, a.attnotnull, a.atttypid + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + WHERE c.oid = reloid + AND NOT a.attisdropped + AND a.attname = const.pkey; + + -- Found something named right... + IF FOUND THEN + + -- And it's an integer column... + IF rec.atttypid IN (20,21,23) THEN + + -- And it's a unique primary key! Done! + IF (rec.indisprimary OR rec.indisunique) AND rec.attnotnull THEN + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey); + RETURN true; + + -- Check and see if the column values are unique and not null, + -- if they are, we can use this column... + ELSE + + -- Assume things are OK until proven otherwise... + useable_key := true; + + BEGIN + sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey); + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql; + EXECUTE sql; + EXCEPTION + -- Failed unique check... + WHEN unique_violation THEN + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey); + useable_key := false; + -- Failed not null check... + WHEN not_null_violation THEN + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey); + useable_key := false; + -- Other fatal error + WHEN others THEN + PERFORM _CDB_Error(sql, '_CDB_Has_Usable_Primary_ID'); + END; + + -- Clean up test constraint + IF useable_key THEN + PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey)); + + -- Move non-unique column out of the way + ELSE + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', + Format('found non-unique ''%s'', renaming it', const.pkey); + + PERFORM _CDB_SQL( + Format('ALTER TABLE %s RENAME COLUMN %s TO %I', + reloid::text, rec.attname, + cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), + '_CDB_Has_Usable_Primary_ID'); + + END IF; + + return useable_key; + + END IF; + + -- It's not an integer column, we have to rename it + ELSE + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', + Format('found non-integer ''%s'', renaming it', const.pkey); + + PERFORM _CDB_SQL( + Format('ALTER TABLE %s RENAME COLUMN %s TO %I', + reloid::text, rec.attname, cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), + '_CDB_Has_Usable_Primary_ID'); + + END IF; + + -- There's no column there named pkey + ELSE + + -- Is there another suitable primary key already? + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + WHERE c.oid = reloid AND NOT a.attisdropped + AND i.indisprimary AND i.indisunique AND a.attnotnull AND a.atttypid IN (20,21,23); + + -- Yes! Ok, rename it. + IF FOUND THEN + PERFORM _CDB_SQL(Format('ALTER TABLE %s RENAME COLUMN %s TO %s', reloid::text, rec.attname, const.pkey),'_CDB_Has_Usable_Primary_ID'); + RETURN true; + ELSE + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', + Format('found no useful column for ''%s''', const.pkey); + END IF; + + END IF; + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 'function complete'; + + -- Didn't find re-usable key, so return FALSE + RETURN false; + +END; +$$ LANGUAGE 'plpgsql'; + + +CREATE OR REPLACE FUNCTION _CDB_Has_Usable_PK_Sequence(reloid REGCLASS) +RETURNS BOOLEAN +AS $$ +DECLARE + seq TEXT; + const RECORD; + has_sequence BOOLEAN = false; +BEGIN + + const := _CDB_Columns(); + + SELECT pg_get_serial_sequence(reloid::text, const.pkey) + INTO STRICT seq; + has_sequence := seq IS NOT NULL; + + RETURN has_sequence; +END; +$$ LANGUAGE 'plpgsql'; + +-- Return a set of columns that can be candidates to be the_geom[webmercator] +-- with some extra information to analyze them. +CREATE OR REPLACE FUNCTION _cdb_geom_candidate_columns(reloid REGCLASS) +RETURNS TABLE (attname name, srid integer, typname name, desired_attname text, desired_srid integer) +AS $$ +DECLARE + const RECORD; +BEGIN + + const := _CDB_Columns(); + + RETURN QUERY + SELECT + a.attname, + CASE WHEN t.typname = 'geometry' THEN postgis_typmod_srid(a.atttypmod) ELSE NULL END AS srid, + t.typname, + f.desired_attname, f.desired_srid + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid, + (VALUES (const.geomcol, 4326), (const.mercgeomcol, 3857) ) as f(desired_attname, desired_srid) + WHERE c.oid = reloid + AND a.attnum > 0 + AND NOT a.attisdropped + AND postgis_typmod_srid(a.atttypmod) IN (4326, 3857, 0) + ORDER BY t.oid ASC; +END; +$$ LANGUAGE 'plpgsql'; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record') THEN + CREATE TYPE _cdb_has_usable_geom_record + AS (has_usable_geoms boolean, + text_geom_column boolean, + text_geom_column_name text, + text_geom_column_srid boolean, + has_geom boolean, + has_geom_name text, + has_mercgeom boolean, + has_mercgeom_name text); + END IF; +END$$; + +DROP FUNCTION IF EXISTS _CDB_Has_Usable_Geom(REGCLASS); +CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Geom(reloid REGCLASS) +RETURNS _cdb_has_usable_geom_record +AS $$ +DECLARE + r1 RECORD; + r2 RECORD; + rv RECORD; + + const RECORD; + + has_geom BOOLEAN := false; + has_mercgeom BOOLEAN := false; + has_geom_name TEXT; + has_mercgeom_name TEXT; + + -- In case 'the_geom' is a text column + text_geom_column BOOLEAN := false; + text_geom_column_name TEXT := ''; + text_geom_column_srid BOOLEAN := true; + + -- Utility variables + srid INTEGER; + str TEXT; + sql TEXT; +BEGIN + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', 'entered function'; + + -- Read in the names of the CartoDB columns + const := _CDB_Columns(); + + -- Do we have a column we can use? + FOR r1 IN + SELECT * FROM _cdb_geom_candidate_columns(reloid) + LOOP + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('checking column ''%s''', r1.attname); + + -- Name collision: right name (the_geom, the_geomwebmercator?) but wrong type... + IF r1.typname != 'geometry' AND r1.attname = r1.desired_attname THEN + + -- Maybe it's a geometry column hiding in a text column? + IF r1.typname IN ('text','varchar','char') THEN + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('column ''%s'' is a text column', r1.attname); + + BEGIN + sql := Format('SELECT Max(ST_SRID(%I::geometry)) AS srid FROM %I', r1.attname, reloid::text); + EXECUTE sql INTO srid; + -- This gets skipped if EXCEPTION happens + -- Let the table writer know we need to convert from text + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('column ''%s'' can be cast from text to geometry', r1.attname); + text_geom_column := true; + text_geom_column_name := r1.attname; + -- Let the table writer know we need to force an SRID + IF srid = 0 THEN + text_geom_column_srid := false; + END IF; + -- Nope, the text in the column can't be converted into geometry + -- so rename it out of the way + EXCEPTION + WHEN others THEN + IF SQLERRM = 'parse error - invalid geometry' THEN + text_geom_column := false; + str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); + sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); + PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', + Format('Text column %s is not convertible to geometry, renamed to %s', r1.attname, str); + ELSE + RAISE EXCEPTION 'CDB(_CDB_Has_Usable_Geom) UNEXPECTED ERROR'; + END IF; + END; + + -- Just change its name so we can write a new column into that name. + ELSE + str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); + sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); + PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', + Format('%s is the wrong type, renamed to %s', r1.attname, str); + END IF; + + -- Found a geometry column! + ELSIF r1.typname = 'geometry' THEN + + -- If it's the right SRID, we can use it in place without + -- transforming it! + IF r1.srid = r1.desired_srid THEN + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('found acceptable ''%s''', r1.attname); + + IF r1.desired_attname = const.geomcol THEN + has_geom := true; + has_geom_name := r1.attname; + ELSIF r1.desired_attname = const.mercgeomcol THEN + has_mercgeom := true; + has_mercgeom_name := r1.attname; + END IF; + + -- If it's an unknown SRID, we need to know that too + ELSIF r1.srid = 0 THEN + + -- Unknown SRID, we'll have to fill it in later + text_geom_column_srid := true; + + END IF; + + END IF; + + END LOOP; + + SELECT + -- If table is perfect (no transforms required), return TRUE! + has_geom AND has_mercgeom AS has_usable_geoms, + -- If the geometry column is hiding in a text field, return enough info to deal w/ it. + text_geom_column, text_geom_column_name, text_geom_column_srid, + -- Return enough info to rename geom columns if needed + has_geom, has_geom_name, has_mercgeom, has_mercgeom_name + INTO rv; + + RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('returning %s', rv); + + RETURN rv; + +END; +$$ LANGUAGE 'plpgsql'; + + +-- Create a copy of the table. Assumes that the "Has usable" functions +-- have already been run, so that if there is a 'cartodb_id' column, it is +-- a "good" one, and the same for the geometry columns. If all the required +-- columns are in place already, it no-ops and just renames the table to +-- the destination if necessary. +CREATE OR REPLACE FUNCTION _CDB_Rewrite_Table(reloid REGCLASS, destschema TEXT DEFAULT NULL) +RETURNS BOOLEAN +AS $$ +DECLARE + + relname TEXT; + relschema TEXT; + + destoid REGCLASS; + destname TEXT; + destseq TEXT; + destseqmax INTEGER; + + copyname TEXT; + + column_name_sql TEXT; + geom_transform_sql TEXT := NULL; + geom_column_source TEXT := ''; + + rec RECORD; + const RECORD; + gc RECORD; + sql TEXT; + str TEXT; + table_srid INTEGER; + geom_srid INTEGER; + + has_usable_primary_key BOOLEAN; + has_usable_pk_sequence BOOLEAN; + +BEGIN + + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', 'entered function'; + + -- Read CartoDB standard column names in + const := _CDB_Columns(); + + -- Save the raw schema/table names for later + SELECT n.nspname, c.relname, c.relname + INTO STRICT relschema, relname, destname + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = reloid; + + -- Default the destination to current schema if unspecified + IF destschema IS NULL THEN + destschema := relschema; + END IF; + + -- See if there is a primary key column we need to carry along to the + -- new table. If this is true, it implies there is an indexed + -- primary key of integer type named (by default) cartodb_id + SELECT _CDB_Has_Usable_Primary_ID(reloid) + INTO STRICT has_usable_primary_key; + + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_primary_key %', has_usable_primary_key; + + -- See if the candidate primary key column has a sequence for default + -- values. No usable pk implies has_usable_pk_sequence = false. + has_usable_pk_sequence := false; + IF has_usable_primary_key THEN + SELECT _CDB_Has_Usable_PK_Sequence(reloid) + INTO STRICT has_usable_pk_sequence; + END IF; + + -- See if the geometry columns we need are already available + -- on the table. If they are, we don't need to do any bulk + -- transformation of the table, we can just ensure proper + -- indexes are in place and apply a rename + SELECT * + FROM _CDB_Has_Usable_Geom(reloid) + INTO STRICT gc; + + -- If geom is the wrong name, just rename it. + IF gc.has_geom AND gc.has_geom_name != const.geomcol THEN + sql := Format('ALTER TABLE %s DROP COLUMN IF EXISTS %I', reloid::text, const.geomcol); + PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table'); + sql := Format('ALTER TABLE %s RENAME COLUMN %I TO %I', reloid::text, gc.has_geom_name, const.geomcol); + PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table'); + END IF; + + -- If mercgeom is the wrong name, just rename it. + IF gc.has_mercgeom AND gc.has_mercgeom_name != const.mercgeomcol THEN + sql := Format('ALTER TABLE %s DROP COLUMN IF EXISTS %I', reloid::text, const.mercgeomcol); + PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table'); + sql := Format('ALTER TABLE %s RENAME COLUMN %I TO %I', reloid::text, gc.has_mercgeom_name, const.mercgeomcol); + PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table'); + END IF; + + + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_geoms %', gc.has_usable_geoms; + + -- We can only avoid a rewrite if both the key and + -- geometry are usable + + -- No table re-write is required, BUT a rename is required to + -- a destination schema, so do that now + IF has_usable_primary_key AND has_usable_pk_sequence AND gc.has_usable_geoms THEN + IF destschema != relschema THEN + + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table needs to be moved to schema (%)', destschema; + PERFORM _CDB_SQL(Format('ALTER TABLE %s SET SCHEMA %I', reloid::text, destschema), '_CDB_Rewrite_Table'); + + ELSE + + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table in the perfect place'; + + END IF; + + RETURN true; + + END IF; + + -- We must rewrite, so here we go... + + + -- Put the primary key sequence in the right schema + -- If the new table is not moving, better ensure the sequence name + -- is unique + destseq := cartodb._CDB_Unique_Identifier(NULL, relname, '_' || const.pkey || '_seq', destschema); + destseq := Format('%I.%I', destschema, destseq); + PERFORM _CDB_SQL(Format('CREATE SEQUENCE %s', destseq), '_CDB_Rewrite_Table'); + + -- Temporary table name if we are re-writing in place + -- Note copyname is already escaped and safe to use as identifier + IF destschema = relschema THEN + copyname := Format('%I.%I', destschema, cartodb._CDB_Unique_Identifier(NULL, destname, NULL), destschema); + ELSE + copyname := Format('%I.%I', destschema, destname); + END IF; + + -- Start building the SQL! + sql := Format('CREATE TABLE %s AS SELECT ', copyname); + + -- Add cartodb ID! + IF has_usable_primary_key THEN + sql := sql || const.pkey; + ELSE + sql := sql || 'nextval(''' || destseq || ''') AS ' || const.pkey; + END IF; + + -- Add the geometry columns! + IF gc.has_usable_geoms THEN + sql := sql || ',' || const.geomcol || ',' || const.mercgeomcol; + ELSE + + -- Arg, this "geometry" column is actually text!! + -- OK, we tested back in our geometry column research that it could + -- be safely cast to geometry, so let's do that. + IF gc.text_geom_column THEN + + WITH t AS ( + SELECT + a.attname, + CASE WHEN NOT gc.text_geom_column_srid THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start, + CASE WHEN NOT gc.text_geom_column_srid THEN ',4326)' ELSE '' END AS missing_srid_end + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid + WHERE c.oid = reloid + AND t.typname IN ('text','varchar','char') + AND a.attnum > 0 + AND a.attname = gc.text_geom_column_name + AND NOT a.attisdropped + ORDER BY a.attnum + LIMIT 1 + ) + SELECT ', ST_Transform(' + || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end + || ',4326)::Geometry(GEOMETRY,4326) AS ' + || const.geomcol + || ', cartodb.CDB_TransformToWebmercator(' + || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end + || ')::Geometry(GEOMETRY,3857) AS ' + || const.mercgeomcol, + t.attname + INTO geom_transform_sql, geom_column_source + FROM t; + + IF NOT FOUND THEN + -- We checked that this column existed already, it bloody well + -- better be found. + RAISE EXCEPTION 'CDB(_CDB_Rewrite_Table): Text column % is missing!', gc.text_geom_column_name; + ELSE + sql := sql || geom_transform_sql; + END IF; + + -- There is at least one true geometry column in here, we'll + -- reproject that into the projections we need. + ELSE + + -- Find the column we are going to be working with (the first + -- column with type "geometry") + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid + WHERE c.oid = reloid + AND t.typname = 'geometry' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum + LIMIT 1; + + -- The SRID could be undeclared at the table level, but still + -- exist in the geometries themselves. We first find our geometry + -- column and read the first SRID off it it, if there is a row + -- to read. + IF FOUND THEN + EXECUTE Format('SELECT ST_SRID(%s) AS srid FROM %s LIMIT 1', rec.attname, reloid::text) + INTO geom_srid; + ELSE + geom_srid := 0; + END IF; + + -- The geometry columns weren't in the right projection, + -- so we need to find the first decent geometry column + -- in the table and wrap it in two transforms, one to 4326 + -- and another to 3857. Then remember its name so we can + -- ignore it when we build the list of other columns to + -- add to the output table + WITH t AS ( + SELECT + a.attname, + postgis_typmod_type(a.atttypmod) AS geomtype, + CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start, + CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN ',4326)' ELSE '' END AS missing_srid_end + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid, + ( SELECT geom_srid AS srid ) AS srid + WHERE c.oid = reloid + AND t.typname = 'geometry' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum + LIMIT 1 + ) + SELECT ', ST_Transform(' + || t.missing_srid_start || t.attname || t.missing_srid_end + || ',4326)::Geometry(GEOMETRY,4326) AS ' + || const.geomcol + || ', cartodb.CDB_TransformToWebmercator(' + || t.missing_srid_start || t.attname || t.missing_srid_end + || ')::Geometry(GEOMETRY,3857) AS ' + || const.mercgeomcol, + t.attname + INTO geom_transform_sql, geom_column_source + FROM t; + + IF NOT FOUND THEN + -- If there are no geometry columns, we continue making a + -- non-spatial table. This is important for folks who want + -- their tables to invalidate the SQL API + -- cache on update/insert/delete. + geom_column_source := ''; + sql := sql || ',NULL::geometry(Geometry,4326) AS ' || const.geomcol; + sql := sql || ',NULL::geometry(Geometry,3857) AS ' || const.mercgeomcol; + ELSE + sql := sql || geom_transform_sql; + END IF; + + END IF; + + END IF; + + -- Add now add all the rest of the columns + -- by selecting their names into an array and + -- joining the array with a comma + SELECT + ',' || array_to_string(array_agg(Format('%I',a.attname)),',') AS column_name_sql, + Count(*) AS count + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_type t ON a.atttypid = t.oid + WHERE c.oid = reloid + AND a.attnum > 0 + AND a.attname NOT IN (const.geomcol, const.mercgeomcol, const.pkey, geom_column_source) + AND NOT a.attisdropped; + + + -- No non-cartodb columns? Possible, I guess. + IF rec.count = 0 THEN + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', 'found no extra columns'; + column_name_sql := ''; + ELSE + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', Format('found extra columns columns ''%s''', rec.column_name_sql); + column_name_sql := rec.column_name_sql; + END IF; + + -- Add the source table to the SQL + sql := sql || column_name_sql || ' FROM ' || reloid::text; + RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', sql; + + -- Run it! + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + + -- Set up the primary key sequence + -- If we copied the primary key from the original data, we need + -- to set the sequence to the maximum value of that key + EXECUTE Format('SELECT max(%s) FROM %s', + const.pkey, copyname) + INTO destseqmax; + + IF destseqmax IS NOT NULL THEN + PERFORM _CDB_SQL(Format('SELECT setval(''%s'', %s)', destseq, destseqmax), '_CDB_Rewrite_Table'); + END IF; + + -- Make the primary key use the sequence as its default value + sql := Format('ALTER TABLE %s ALTER COLUMN %s SET DEFAULT nextval(''%s'')', + copyname, const.pkey, destseq); + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + + -- Make the sequence owned by the table, so when the table drops, + -- the sequence does too + sql := Format('ALTER SEQUENCE %s OWNED BY %s.%s', destseq, copyname, const.pkey); + PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table'); + + + -- We just made a copy, so we can drop the original now + sql := Format('DROP TABLE %s', reloid::text); + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + + -- If the table is being created by a SECURITY DEFINER function + -- make sure the user is set back to the user who is connected + IF current_user != session_user THEN + sql := Format('ALTER TABLE IF EXISTS %s OWNER TO %s', copyname, session_user); + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + sql := Format('ALTER SEQUENCE IF EXISTS %s OWNER TO %s', destseq, session_user); + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + END IF; + + -- If we used a temporary destination table + -- we can now rename it into place + IF destschema = relschema THEN + sql := Format('ALTER TABLE %s RENAME TO %I', copyname, destname); + PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table'); + END IF; + + RETURN true; + +END; +$$ LANGUAGE 'plpgsql'; + + +-- Assumes the table already has the right metadata columns +-- (primary key and two geometry columns) and adds primary key +-- and geometry indexes if necessary. +CREATE OR REPLACE FUNCTION _CDB_Add_Indexes(reloid REGCLASS) + RETURNS BOOLEAN +AS $$ +DECLARE + rec RECORD; + const RECORD; + iname TEXT; + sql TEXT; + relname TEXT; +BEGIN + + RAISE DEBUG 'CDB(_CDB_Add_Indexes): %', 'entered function'; + + -- Read CartoDB standard column names in + const := _CDB_Columns(); + + -- Extract just the relname to use for the index names + SELECT c.relname + INTO STRICT relname + FROM pg_class c + WHERE c.oid = reloid; + + -- Is there already a primary key on this table for + -- a column other than our chosen primary key? + SELECT ci.relname AS pkey + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + JOIN pg_class ci ON i.indexrelid = ci.oid + WHERE c.oid = reloid + AND NOT a.attisdropped + AND a.attname != const.pkey + AND i.indisprimary; + + -- Yes? Then drop it, we're adding our own PK to the column + -- we prefer. + IF FOUND THEN + RAISE DEBUG 'CDB(_CDB_Add_Indexes): dropping unwanted primary key ''%''', rec.pkey; + sql := Format('ALTER TABLE %s DROP CONSTRAINT IF EXISTS %s', reloid::text, rec.pkey); + PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes'); + END IF; + + + -- Is the default primary key flagged as primary? + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + JOIN pg_class ci ON ci.oid = i.indexrelid + WHERE attnum > 0 + AND c.oid = reloid + AND a.attname = const.pkey + AND i.indisprimary + AND i.indisunique + AND NOT attisdropped; + + -- No primary key? Add one. + IF NOT FOUND THEN + sql := Format('ALTER TABLE %s ADD PRIMARY KEY (%s)', reloid::text, const.pkey); + PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes'); + END IF; + + -- Add geometry indexes to all "special geometry columns" that + -- don't have one (either have no index at all, or have a non-GIST index) + FOR rec IN + SELECT a.attname, n.nspname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 + LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + WHERE NOT attisdropped + AND a.attname IN (const.geomcol, const.mercgeomcol) + AND c.oid = reloid + AND i.indexrelid IS NULL + UNION + SELECT a.attname, n.nspname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 + JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey) + JOIN pg_class ci ON ci.oid = i.indexrelid + JOIN pg_am am ON ci.relam = am.oid + WHERE NOT attisdropped + AND a.attname IN (const.geomcol, const.mercgeomcol) + AND c.oid = reloid + AND am.amname != 'gist' + LOOP + sql := Format('CREATE INDEX ON %s USING GIST (%s)', reloid::text, rec.attname); + PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes'); + END LOOP; + + RETURN true; + +END; +$$ LANGUAGE 'plpgsql'; + +DROP FUNCTION IF EXISTS CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS); +CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS) +RETURNS REGCLASS +AS $$ +DECLARE + + is_raster BOOLEAN; + relname TEXT; + relschema TEXT; + + destoid REGCLASS; + destname TEXT; + + rec RECORD; + +BEGIN + + -- Save the raw schema/table names for later + SELECT n.nspname, c.relname, c.relname + INTO STRICT relschema, relname, destname + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = reloid; + + PERFORM cartodb._CDB_check_prerequisites(destschema, reloid); + + -- Check destination schema exists + -- Throws an exception of there is no matching schema + IF destschema IS NOT NULL THEN + + SELECT n.nspname + INTO rec FROM pg_namespace n WHERE n.nspname = destschema; + IF NOT FOUND THEN + RAISE EXCEPTION 'Schema ''%'' does not exist', destschema; + END IF; + + ELSE + destschema := relschema; + END IF; + + -- Drop triggers first + PERFORM _CDB_drop_triggers(reloid); + + -- Rasters only get a cartodb_id and a limited selection of triggers + -- underlying assumption is that they are already formed up correctly + SELECT cartodb._CDB_is_raster_table(destschema, reloid) INTO is_raster; + IF is_raster THEN + + PERFORM cartodb._CDB_create_cartodb_id_column(reloid); + PERFORM cartodb._CDB_create_raster_triggers(destschema, reloid); + + ELSE + + -- Rewrite (or rename) the table to the new location + PERFORM _CDB_Rewrite_Table(reloid, destschema); + + -- The old regclass might not be valid anymore if we re-wrote the table... + destoid := (destschema || '.' || destname)::regclass; + + -- Add indexes to the destination table, as necessary + PERFORM _CDB_Add_Indexes(destoid); + + -- Add triggers to the destination table, as necessary + PERFORM _CDB_create_triggers(destschema, destoid); + + END IF; + + RETURN (destschema || '.' || destname)::regclass; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/test/support/CDB_ColumnNames.sql b/test/support/CDB_ColumnNames.sql new file mode 100644 index 00000000..ce742164 --- /dev/null +++ b/test/support/CDB_ColumnNames.sql @@ -0,0 +1,18 @@ +-- Function returning the column names of a table +CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS) +RETURNS SETOF information_schema.sql_identifier +AS $$ + + SELECT c.column_name + FROM information_schema.columns c, pg_class _tn, pg_namespace _sn + WHERE table_name = _tn.relname + AND table_schema = _sn.nspname + AND _tn.oid = $1::oid + AND _sn.oid = _tn.relnamespace + ORDER BY ordinal_position; + +$$ LANGUAGE SQL; + +-- This is to migrate from pre-0.2.0 version +-- See http://github.com/CartoDB/cartodb-postgresql/issues/36 +GRANT EXECUTE ON FUNCTION CDB_ColumnNames(REGCLASS) TO PUBLIC; diff --git a/test/support/CDB_ForeignTable.sql b/test/support/CDB_ForeignTable.sql new file mode 100644 index 00000000..b03d284b --- /dev/null +++ b/test/support/CDB_ForeignTable.sql @@ -0,0 +1,199 @@ +--------------------------- +-- FDW MANAGEMENT FUNCTIONS +-- +-- All the FDW settings are read from the `cdb_conf.fdws` entry json file. +--------------------------- + +CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text, config json) +RETURNS void +AS $$ +DECLARE + row record; + option record; + org_role text; +BEGIN + -- This function tries to be as idempotent as possible, by not creating anything more than once + -- (not even using IF NOT EXIST to avoid throwing warnings) + IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN + CREATE EXTENSION postgres_fdw; + END IF; + -- Create FDW first if it does not exist + IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name) + THEN + EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name); + END IF; + + -- Set FDW settings + FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p + LOOP + IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key) + THEN + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value); + ELSE + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value); + END IF; + END LOOP; + + -- Create user mappings + FOR row IN SELECT p.key, p.value from lateral json_each(config->'users') p LOOP + -- Check if entry on pg_user_mappings exists + + IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN + EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name); + END IF; + + -- Update user mapping settings + FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP + IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value); + ELSE + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value); + END IF; + END LOOP; + END LOOP; + + -- Create schema if it does not exist. + IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN + EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name); + END IF; + + -- Give the organization role usage permisions over the schema + SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() INTO org_role; + EXECUTE FORMAT ('GRANT USAGE ON SCHEMA %I TO %I', fdw_name, org_role); + + -- Bring here the remote cdb_tablemetadata + IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN + EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name); + END IF; + EXECUTE FORMAT ('GRANT SELECT ON %I.cdb_tablemetadata TO %I', fdw_name, org_role); + +END +$$ +LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDWS() +RETURNS VOID AS +$$ +DECLARE +row record; +BEGIN + FOR row IN SELECT p.key, p.value from lateral json_each(cartodb.CDB_Conf_GetConf('fdws')) p LOOP + EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING row.key, row.value; + END LOOP; + END +$$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text) + RETURNS void AS +$BODY$ +DECLARE +config json; +BEGIN + SELECT p.value FROM LATERAL json_each(cartodb.CDB_Conf_GetConf('fdws')) p WHERE p.key = fdw_name INTO config; + EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING fdw_name, config; +END +$BODY$ +LANGUAGE plpgsql VOLATILE; + +CREATE OR REPLACE FUNCTION cartodb.CDB_Add_Remote_Table(source text, table_name text) + RETURNS void AS +$$ +BEGIN + PERFORM cartodb._CDB_Setup_FDW(source); + EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', source, table_name, source, source); + --- Grant SELECT to publicuser + EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO publicuser;', source, table_name); +END +$$ +LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cartodb.CDB_Get_Foreign_Updated_At(foreign_table regclass) + RETURNS timestamp with time zone AS +$$ +DECLARE + remote_table_name text; + fdw_schema_name text; + time timestamp with time zone; +BEGIN + -- This will turn a local foreign table (referenced as regclass) to its fully qualified text remote table reference. + WITH a AS (SELECT ftoptions FROM pg_foreign_table WHERE ftrelid=foreign_table LIMIT 1), + b as (SELECT (pg_options_to_table(ftoptions)).* FROM a) + SELECT FORMAT('%I.%I', (SELECT option_value FROM b WHERE option_name='schema_name'), (SELECT option_value FROM b WHERE option_name='table_name')) + INTO remote_table_name; + + -- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table. + SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name; + EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time; + RETURN time; +END +$$ +LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid) +RETURNS TEXT AS $$ + SELECT option_value FROM pg_options_to_table(( + + SELECT fs.srvoptions + FROM pg_foreign_table ft + LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid + WHERE ft.ftrelid = reloid + + )) WHERE option_name='dbname'; +$$ LANGUAGE SQL; + + +-- Return a set of (dbname, schema_name, table_name, updated_at) +-- It is aware of foreign tables +-- It assumes the local (schema_name, table_name) map to the remote ones with the same name +-- Note: dbname is never quoted whereas schema and table names are when needed. +CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text) +RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz) +AS $$ + WITH query_tables AS ( + SELECT unnest(CDB_QueryTablesText(query)) schema_table_name + ), query_tables_oid AS ( + SELECT schema_table_name, schema_table_name::regclass::oid AS reloid + FROM query_tables + ), + fqtn AS ( + SELECT + (CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid) + ELSE current_database() + END)::text AS dbname, + quote_ident(n.nspname::text) schema_name, + quote_ident(c.relname::text) table_name, + c.relkind, + query_tables_oid.reloid + FROM query_tables_oid, pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = query_tables_oid.reloid + ) + SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, + (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) + END) AS updated_at + FROM fqtn; +$$ LANGUAGE SQL; + + +-- Return the last updated time of a set of tables +-- It is aware of foreign tables +-- It assumes the local (schema_name, table_name) map to the remote ones with the same name +CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[]) +RETURNS timestamptz AS $$ + WITH t AS ( + SELECT unnest(tables) AS schema_table_name + ), t_oid AS ( + SELECT (t.schema_table_name)::regclass::oid as reloid FROM t + ), t_updated_at AS ( + SELECT + (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) + END) AS updated_at + FROM t_oid + LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid + ) SELECT max(updated_at) FROM t_updated_at; +$$ LANGUAGE SQL; diff --git a/test/support/CDB_Overviews.sql b/test/support/CDB_Overviews.sql new file mode 100644 index 00000000..53be5117 --- /dev/null +++ b/test/support/CDB_Overviews.sql @@ -0,0 +1,683 @@ +-- security definer + +-- Pattern that can be used to detect overview tables and Extract +-- the intended zoom level from the table name. +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator() +RETURNS TEXT +AS $$ + BEGIN + RETURN '\A_vovw_(\d+)_'; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; +-- substring(tablename from _CDB_OverviewTableDiscriminator()) + + +-- Pattern matched by the overview tables of a given base table name. +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT) +RETURNS TEXT +AS $$ + BEGIN + RETURN _CDB_OverviewTableDiscriminator() || base_table; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; +-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table) + +-- Name of an overview table, given the base table name and the Z level +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER) +RETURNS TEXT +AS $$ + BEGIN + RETURN '_vovw_' || z::text || '_' || base_table; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Condition to check if a tabla is an overview table of some base table +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT) +RETURNS BOOLEAN +AS $$ + BEGIN + RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table); + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Extract the Z level from an overview table name +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT) +RETURNS INTEGER +AS $$ + BEGIN + RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Name of the base table corresponding to an overview table +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT) +RETURNS TEXT +AS $$ + BEGIN + IF _CDB_OverviewTableZ(overview_table) IS NULL THEN + RETURN overview_table; + ELSE + RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), ''); + END IF; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + + +-- Remove a dataset's existing overview tables. +-- Scope: public +-- Parameters: +-- reloid: oid of the table. +CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS) +RETURNS void +AS $$ +DECLARE + row record; +BEGIN + FOR row IN + SELECT * FROM CDB_Overviews(reloid) + LOOP + EXECUTE Format('DROP TABLE %s;', row.overview_table); + RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table; + END LOOP; +END; +$$ LANGUAGE PLPGSQL VOLATILE; + + + +-- Return existing overviews (if any) for a given dataset table +-- Scope: public +-- Parameters +-- reloid: oid of the input table. +-- Return relation of overviews for the table with +-- the base table oid, +-- z level of the overview and overview table oid, ordered by z. +CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS) +RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS) +AS $$ + -- FIXME: this will fail if the overview tables + -- require a explicit schema name + -- possible solutions: return table names as text instead of regclass + -- or add schema of reloid before casting to regclass + SELECT + reloid AS base_table, + _CDB_OverviewTableZ(cdb_usertables) AS z, + cdb_usertables::regclass AS overview_table + FROM CDB_UserTables() + WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables) + ORDER BY z; +$$ LANGUAGE SQL; + +-- Return existing overviews (if any) for multiple dataset tables. +-- Scope: public +-- Parameters +-- tables: Array of input tables oids +-- Return relation of overviews for the table with +-- the base table oid, +-- z level of the overview and overview table oid, ordered by z. +-- Note: CDB_Overviews can be applied to the result of CDB_QueryTablesText +-- to obtain the overviews applicable to a query. +CREATE OR REPLACE FUNCTION CDB_Overviews(tables regclass[]) +RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS) +AS $$ + SELECT + base_table::regclass AS base_table, + _CDB_OverviewTableZ(cdb_usertables) AS z, + cdb_usertables::regclass AS overview_table + FROM + CDB_UserTables(), unnest(tables) base_table + WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables) + ORDER BY base_table, z; +$$ LANGUAGE SQL; + +-- Schema and relation names of a table given its reloid +-- Scope: private. +-- Parameters +-- reloid: oid of the table. +-- Return (schema_name, table_name) +-- note that returned names will be quoted if necessary +CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT) +AS $$ + BEGIN + SELECT n.nspname, c.relname + INTO STRICT schema_name, table_name + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = reloid; + END +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Calculate the estimated extent of a cartodbfy'ed table. +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. +-- Return value A box2d extent in 3857. +CREATE OR REPLACE FUNCTION _cdb_estimated_extent(reloid REGCLASS) +RETURNS box2d +AS $$ + DECLARE + ext box2d; + ext_query text; + table_id record; + BEGIN + + SELECT n.nspname AS schema_name, c.relname table_name INTO STRICT table_id + FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid; + + ext_query = format( + 'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');', + table_id.schema_name, table_id.table_name, 'the_geom_webmercator' + ); + + BEGIN + EXECUTE ext_query INTO ext; + EXCEPTION + -- This is the typical ERROR: stats for "mytable" do not exist + WHEN internal_error THEN + -- Get stats and execute again + EXECUTE format('ANALYZE %1$I', reloid); + EXECUTE ext_query INTO ext; + END; + + RETURN ext; + END; +$$ LANGUAGE PLPGSQL VOLATILE; + +-- Determine the max feature density of a given dataset. +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- nz: number of zoom levels to consider from z0 upward. +-- Return value: feature density (num_features / webmercator_squared_meters). +CREATE OR REPLACE FUNCTION _CDB_Feature_Density(reloid REGCLASS, nz integer) +RETURNS FLOAT8 +AS $$ + DECLARE + fd FLOAT8; + min_features TEXT; + n integer = 4; + c FLOAT8; + BEGIN + -- TODO: for small total count or extents we could just: + -- EXECUTE 'SELECT Count(*)/ST_Area(ST_Extent(the_geom_webmercator)) FROM ' || reloid::text || ';' INTO fd; + + -- min_features is a SQL subexpression which can depend on z and represents + -- the minimum number of features to recursively consider a tile. + -- We can either use a fixed minimum number of features per tile + -- or a minimum feature density by dividing the number of features by + -- the area of tiles at level Z: c*c*power(2, -2*z) + -- with c = CDB_XYZ_Resolution(-8) (earth circumference) + min_features = '500'; + SELECT CDB_XYZ_Resolution(-8) INTO c; + + -- We first compute a set of *seed* tiles, of the minimum Z level, z0, such that + -- they cover the extent of the table and we have at least n of them in each + -- linear dimension (i.e. at least n*n tiles cover the extent). + -- We compute the number of features in these tiles, and recursively in + -- subtiles up to level z0 + nz. Then we compute the maximum of the feature + -- density (per tile area in webmercator squared meters) for all the + -- considered tiles. + EXECUTE Format(' + WITH RECURSIVE t(x, y, z, e) AS ( + WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g), + base AS ( + SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z + FROM ext + ), + lim AS ( + SELECT + FLOOR((ST_XMin(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x0, + FLOOR((ST_XMax(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x1, + FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMin(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y1, + FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMax(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y0 + FROM ext, base + ), + seed AS ( + SELECT xt, yt, base.z, ( + SELECT count(*) FROM %1$s + WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z) + ) e + FROM base, lim, generate_series(lim.x0, lim.x1) xt, generate_series(lim.y0, lim.y1) yt + ) + SELECT * from seed + UNION ALL + SELECT x*2 + xx, y*2 + yy, t.z+1, ( + SELECT count(*) FROM %1$s + WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1) + ) + FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy) + WHERE t.e > %2$s AND t.z < (base.z + %3$s) + ) + SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0; + ', reloid::text, min_features, nz, n, c, reloid::oid) + INTO fd; + RETURN fd; + END +$$ LANGUAGE PLPGSQL STABLE; + +-- Experimental default strategy to assign a reference base Z level +-- to a cartodbfied table. The resulting Z level represents the +-- minimum scale level at which the table data can be rendered +-- without overcrowded results or loss of detail. +-- Parameters: +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- Return value: Z level as an integer +CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS) +RETURNS INTEGER +AS $$ + DECLARE + lim FLOAT8 := 500; -- TODO: determine/parameterize this + nz integer := 4; + fd FLOAT8; + c FLOAT8; + BEGIN + -- Compute fd as an estimation of the (maximum) number + -- of features per unit of tile area (in webmercator squared meters) + SELECT _CDB_Feature_Density(reloid, nz) INTO fd; + -- lim maximum number of (desiderable) features per tile + -- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference) + -- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z) + -- => fd*ta(z) if the average number of features per tile at level z + -- find minimum z so that fd*ta(z) <= lim + -- compute a rough 'feature density' value + SELECT CDB_XYZ_Resolution(-8) INTO c; + RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2); + END; +$$ LANGUAGE PLPGSQL STABLE; + +-- Overview table name for a given Z level and base dataset or overview table +-- Scope: private. +-- Parameters: +-- ref reference table (can be the base table of the dataset or an existing +-- overview) from which the overview is being generated. +-- ref_z Z level of the reference table +-- overview_z Z level of the overview to be named, must be smaller than ref_z +-- Return value: the name to be used for the overview. The name is always +-- unqualified (does not include a schema name). +CREATE OR REPLACE FUNCTION _CDB_Overview_Name(ref REGCLASS, ref_z INTEGER, overview_z INTEGER) +RETURNS TEXT +AS $$ + DECLARE + schema_name TEXT; + base TEXT; + suffix TEXT; + is_overview BOOLEAN; + BEGIN + SELECT * FROM _cdb_split_table_name(ref) INTO schema_name, base; + SELECT _CDB_OverviewBaseTableName(base) INTO base; + RETURN _CDB_OverviewTableName(base, overview_z); + END +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Sampling reduction method. +-- Valid for any kind of geometry. +-- Scope: private. +-- reloid original table (can be the base table of the dataset or an existing +-- overview) from which the overview is being generated. +-- ref_z Z level assigned to the original table +-- overview_z Z level of the overview to be generated, must be smaller than ref_z +-- Return value: Name of the generated overview table +CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER) +RETURNS REGCLASS +AS $$ + DECLARE + overview_rel TEXT; + fraction FLOAT8; + base_name TEXT; + class_info RECORD; + num_samples INTEGER; + BEGIN + overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z); + fraction := power(2, 2*(overview_z - ref_z)); + + -- FIXME: handle schema name for overview_rel if reloid requires it + EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel); + + -- Estimate number of rows + SELECT reltuples, relpages FROM pg_class INTO STRICT class_info + WHERE oid = reloid::oid; + + IF class_info.relpages < 2 OR fraction > 0.5 THEN + -- We'll avoid possible CDB_RandomTids problems + EXECUTE Format(' + CREATE TABLE %I AS SELECT * FROM %s WHERE random() < %s; + ', overview_rel, reloid, fraction); + ELSE + num_samples := ceil(class_info.reltuples*fraction); + EXECUTE Format(' + CREATE TABLE %1$I AS SELECT * FROM %2$s + WHERE ctid = ANY ( + ARRAY[ + (SELECT CDB_RandomTids(''%2$s'', %3$s)) + ] + ); + ', overview_rel, reloid, num_samples); + END IF; + + RETURN overview_rel; + END; +$$ LANGUAGE PLPGSQL; + +-- Register new overview table (post-creation chores) +-- Scope: private +-- Parameters: +-- dataset: oid of the input dataset table, It must be a cartodbfy'ed table. +-- overview_table: oid of the overview table to be registered. +-- overview_z: intended Z level for the overview table +-- This function is declared SECURITY DEFINER so it executes with the privileges +-- of the function creator to have a chance to alter the privileges of the +-- overview table to match those of the dataset. It will only perform any change +-- if the overview table belgons to the same scheme as the dataset and it +-- matches the scheme naming for overview tables. +CREATE OR REPLACE FUNCTION _CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER) +RETURNS VOID +AS $$ + DECLARE + sql TEXT; + table_owner TEXT; + dataset_scheme TEXT; + dataset_name TEXT; + overview_scheme TEXT; + overview_name TEXT; + BEGIN + -- This function will only register a table as an overview table if it matches + -- the overviews naming scheme for the dataset and z level and the table belongs + -- to the same scheme as the the dataset + SELECT * FROM _cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name; + SELECT * FROM _cdb_split_table_name(overview_table) INTO overview_scheme, overview_name; + IF dataset_scheme = overview_scheme AND + overview_name = _CDB_OverviewTableName(dataset_name, overview_z) THEN + + -- preserve the owner of the base table + SELECT u.usename + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid) + WHERE c.relname = dataset::text + INTO table_owner; + EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner); + + -- preserve the table privileges + UPDATE pg_class c_to + SET relacl = c_from.relacl + FROM pg_class c_from + WHERE c_from.oid = dataset + AND c_to.oid = overview_table; + + PERFORM _CDB_Add_Indexes(overview_table); + + -- TODO: If metadata about existing overviews is to be stored + -- it should be done here (CDB_Overviews would consume such metadata) + END IF; + END +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +-- Dataset attributes (column names other than the +-- CartoDB primary key and geometry columns) which should be aggregated +-- in aggregated overviews. +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- Return value: set of attribute names +CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes(reloid REGCLASS) +RETURNS SETOF information_schema.sql_identifier +AS $$ + SELECT c FROM CDB_ColumnNames(reloid) c, _CDB_Columns() cdb + WHERE c NOT IN ( + cdb.pkey, cdb.geomcol, cdb.mercgeomcol + ) +$$ LANGUAGE SQL STABLE; + +-- List of dataset attributes to be aggregated in aggregated overview +-- as a comma-separated SQL expression. +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- Return value: SQL subexpression as text +CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes_Expression(reloid REGCLASS) +RETURNS TEXT +AS $$ +DECLARE + attr_list TEXT; +BEGIN + SELECT string_agg(s.c, ',') FROM ( + SELECT * FROM _CDB_Aggregable_Attributes(reloid) c + ) AS s INTO attr_list; + + RETURN attr_list; +END +$$ LANGUAGE PLPGSQL STABLE; + +-- SQL Aggregation expression for a datase attribute +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- column_name: column to be aggregated +-- table_alias: (optional) table qualifier for the column to be aggregated +-- Return SQL subexpression as text with aggregated attribute aliased +-- with its original name. +CREATE OR REPLACE FUNCTION _CDB_Attribute_Aggregation_Expression(reloid REGCLASS, column_name TEXT, table_alias TEXT DEFAULT '') +RETURNS TEXT +AS $$ +DECLARE + column_type TEXT; + qualified_column TEXT; +BEGIN + IF table_alias <> '' THEN + qualified_column := Format('%I.%I', table_alias, column_name); + ELSE + qualified_column := Format('%I', column_name); + END IF; + + column_type := CDB_ColumnType(reloid, column_name); + + CASE column_type + WHEN 'double precision', 'real', 'integer', 'bigint' THEN + RETURN Format('AVG(%s)::' || column_type, qualified_column); + WHEN 'text' THEN + -- TODO: we could define a new aggregate function that returns distinct + -- separated values with a limit, adding ellipsis if more values existed + -- e.g. with '/' as separator and a limit of three: + -- 'A', 'B', 'A', 'C', 'D' => 'A/B/C/...' + -- Other ideas: if value is unique then use it, otherwise use something + -- like '*' or '(varies)' or '(multiple values)', or NULL + -- Using 'string_agg(' || qualified_column || ',''/'')' + -- here causes + RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type; + ELSE + RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type; + END CASE; +END +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- List of dataset aggregated attributes as a comma-separated SQL expression. +-- Scope: private. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- table_alias: (optional) table qualifier for the columns to be aggregated +-- Return value: SQL subexpression as text +CREATE OR REPLACE FUNCTION _CDB_Aggregated_Attributes_Expression(reloid REGCLASS, table_alias TEXT DEFAULT '') +RETURNS TEXT +AS $$ +DECLARE + attr_list TEXT; +BEGIN + SELECT string_agg(_CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',') + FROM ( + SELECT * FROM _CDB_Aggregable_Attributes(reloid) c + ) AS s INTO attr_list; + + RETURN attr_list; +END +$$ LANGUAGE PLPGSQL STABLE; + +-- Array of geometry types detected in a cartodbfied table +-- For effciency only look at a limited number of rwos. +-- Parameters +-- reloid: oid of the input table. It must be a cartodbfy'ed table. +-- Return value: array of geometry type names +CREATE OR REPLACE FUNCTION _CDB_GeometryTypes(reloid REGCLASS) +RETURNS TEXT[] +AS $$ +DECLARE + gtypes TEXT[]; +BEGIN + EXECUTE Format(' + SELECT array_agg(DISTINCT ST_GeometryType(the_geom)) FROM ( + SELECT the_geom FROM %s + WHERE (the_geom is not null) LIMIT 10 + ) as geom_types + ', reloid) + INTO gtypes; + RETURN gtypes; +END +$$ LANGUAGE PLPGSQL STABLE; + +-- Experimental Overview reduction method for point datasets. +-- It clusters the points using a grid, then aggregates the point in each +-- cluster into a point at the centroid of the clustered records. +-- Scope: private. +-- Parameters: +-- reloid original table (can be the base table of the dataset or an existing +-- overview) from which the overview is being generated. +-- ref_z Z level assigned to the original table +-- overview_z Z level of the overview to be generated, must be smaller than ref_z +-- Return value: Name of the generated overview table +CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER) +RETURNS REGCLASS +AS $$ + DECLARE + overview_rel TEXT; + reduction FLOAT8; + base_name TEXT; + grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z + grid_m FLOAT8; + aggr_attributes TEXT; + attributes TEXT; + columns TEXT; + gtypes TEXT[]; + BEGIN + SELECT _CDB_GeometryTypes(reloid) INTO gtypes; + IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN + -- This strategy only supports datasets with point geomety + RETURN NULL; + RETURN 'x'; + END IF; + + --TODO: check applicability: geometry type, minimum number of points... + + overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z); + + -- compute grid cell size using the overview_z dimension... + SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m; + + attributes := _CDB_Aggregable_Attributes_Expression(reloid); + aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid); + IF attributes <> '' THEN + attributes := ', ' || attributes; + END IF; + IF aggr_attributes <> '' THEN + aggr_attributes := aggr_attributes || ', '; + END IF; + + -- compute the resulting columns in the same order as in the base table + -- cartodb_id, + -- ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom, + -- ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator + -- %4$s + WITH cols AS ( + SELECT + CASE c + WHEN 'cartodb_id' THEN 'cartodb_id' + WHEN 'the_geom' THEN + 'ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom' + WHEN 'the_geom_webmercator' THEN + 'ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator' + ELSE c + END AS column + FROM CDB_ColumnNames(reloid) c + ) + SELECT string_agg(s.column, ',') FROM ( + SELECT * FROM cols + ) AS s INTO columns; + + -- FIXME: handle schema name for overview_rel if reloid requires it + EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel); + + -- Now we cluster the data using a grid of size grid_m + -- and selecte the centroid (average coordinates) of each cluster. + -- If we had a selected numeric attribute of interest we could use it + -- as a weight for the average coordinates. + EXECUTE Format(' + CREATE TABLE %3$I AS + WITH clusters AS ( + SELECT + %5$s + count(*) AS n, + SUM(ST_X(f.the_geom_webmercator)) AS sx, + SUM(ST_Y(f.the_geom_webmercator)) AS sy, + Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx, + Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy, + MIN(cartodb_id) AS cartodb_id + FROM %1$s f + GROUP BY gx, gy + ) + SELECT %6$s FROM clusters + ', reloid::text, grid_m, overview_rel, attributes, aggr_attributes, columns); + + RETURN overview_rel; + END; +$$ LANGUAGE PLPGSQL; + +-- Create overview tables for a dataset. +-- Scope: public +-- Parameters: +-- reloid: oid of the input table. It must be a cartodbfy'ed table with +-- vector features. +-- refscale_strategy: function that computes the reference Z of the dataset +-- reduce_strategy: function that generates overviews from a base table +-- or higher level overview. The overview tables +-- created by the strategy must have the same columns +-- as the base table and in the same order. +-- Return value: Array with the names of the generated overview tables +CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc) +RETURNS text[] +AS $$ +DECLARE + ref_z integer; + overviews_z integer[]; + base_z integer; + base_rel REGCLASS; + overview_z integer; + overview_tables REGCLASS[]; + overviews_step integer := 1; +BEGIN + -- Determine the referece zoom level + EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z; + + -- Determine overlay zoom levels + -- TODO: should be handled by the refscale_strategy? + overview_z := ref_z - 1; + WHILE overview_z >= 0 LOOP + SELECT array_append(overviews_z, overview_z) INTO overviews_z; + overview_z := overview_z - overviews_step; + END LOOP; + + -- Create overlay tables + base_z := ref_z; + base_rel := reloid; + FOREACH overview_z IN ARRAY overviews_z LOOP + EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel; + IF base_rel IS NULL THEN + EXIT; + END IF; + base_z := overview_z; + PERFORM _CDB_Register_Overview(reloid, base_rel, base_z); + SELECT array_append(overview_tables, base_rel) INTO overview_tables; + END LOOP; + + RETURN overview_tables; +END; +$$ LANGUAGE PLPGSQL; diff --git a/test/support/CDB_TableMetadata.sql b/test/support/CDB_TableMetadata.sql new file mode 100644 index 00000000..6f65606c --- /dev/null +++ b/test/support/CDB_TableMetadata.sql @@ -0,0 +1,143 @@ + +CREATE TABLE IF NOT EXISTS + public.CDB_TableMetadata ( + tabname regclass not null primary key, + updated_at timestamp with time zone not null default now() + ); + +CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS + SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at + FROM public.CDB_TableMetadata, pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid; + +-- No one can see this +-- Updates are only possible trough the security definer trigger +-- GRANT SELECT ON public.CDB_TableMetadata TO public; + +-- +-- Trigger logging updated_at in the CDB_TableMetadata +-- and notifying cdb_tabledata_update with table name as payload. +-- +-- Attach to tables like this: +-- +-- CREATE trigger track_updates +-- AFTER INSERT OR UPDATE OR TRUNCATE OR DELETE ON +-- FOR EACH STATEMENT +-- EXECUTE PROCEDURE cdb_tablemetadata_trigger(); +-- +-- NOTE: _never_ attach to CDB_TableMetadata ... +-- +CREATE OR REPLACE FUNCTION CDB_TableMetadata_Trigger() +RETURNS trigger AS +$$ +BEGIN + -- Guard against infinite loop + IF TG_RELID = 'public.CDB_TableMetadata'::regclass::oid THEN + RETURN NULL; + END IF; + + -- Cleanup stale entries + DELETE FROM public.CDB_TableMetadata + WHERE NOT EXISTS ( + SELECT oid FROM pg_class WHERE oid = tabname + ); + + WITH nv as ( + SELECT TG_RELID as tabname, NOW() as t + ), updated as ( + UPDATE public.CDB_TableMetadata x SET updated_at = nv.t + FROM nv WHERE x.tabname = nv.tabname + RETURNING x.tabname + ) + INSERT INTO public.CDB_TableMetadata SELECT nv.* + FROM nv LEFT JOIN updated USING(tabname) + WHERE updated.tabname IS NULL; + + RETURN NULL; +END; +$$ +LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +-- +-- Trigger invalidating varnish whenever CDB_TableMetadata +-- record change. +-- +CREATE OR REPLACE FUNCTION _CDB_TableMetadata_Updated() +RETURNS trigger AS +$$ +DECLARE + tabname TEXT; + rec RECORD; + found BOOL; +BEGIN + + IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN + tabname = NEW.tabname; + ELSE + tabname = OLD.tabname; + END IF; + + -- Notify table data update + -- This needs a little bit more of research regarding security issues + -- see https://github.com/CartoDB/cartodb/pull/241 + -- PERFORM pg_notify('cdb_tabledata_update', tabname); + + --RAISE NOTICE 'Table % was updated', tabname; + + -- This will be needed until we'll have someone listening + -- on the event we just broadcasted: + -- + -- LISTEN cdb_tabledata_update; + -- + + -- Call the first varnish invalidation function owned + -- by a superuser found in cartodb or public schema + -- (in that order) + found := false; + FOR rec IN SELECT u.usesuper, u.usename, n.nspname, p.proname + FROM pg_proc p, pg_namespace n, pg_user u + WHERE p.proname = 'cdb_invalidate_varnish' + AND p.pronamespace = n.oid + AND n.nspname IN ('public', 'cartodb') + AND u.usesysid = p.proowner + AND u.usesuper + ORDER BY n.nspname + LOOP + EXECUTE 'SELECT ' || quote_ident(rec.nspname) || '.' + || quote_ident(rec.proname) + || '(' || quote_literal(tabname) || ')'; + found := true; + EXIT; + END LOOP; + IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF; + + RETURN NULL; +END; +$$ +LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +DROP TRIGGER IF EXISTS table_modified ON CDB_TableMetadata; +-- NOTE: on DELETE we would be unable to convert the table +-- oid (regclass) to its name +CREATE TRIGGER table_modified AFTER INSERT OR UPDATE +ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE + _CDB_TableMetadata_Updated(); + + +-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata +CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass) + RETURNS void AS + $$ + BEGIN + WITH upsert AS ( + UPDATE public.cdb_tablemetadata + SET updated_at = NOW() + WHERE tabname = tablename + RETURNING * + ) + INSERT INTO public.cdb_tablemetadata (tabname, updated_at) + SELECT tablename, NOW() + WHERE NOT EXISTS (SELECT * FROM upsert); + END; + $$ +LANGUAGE 'plpgsql' VOLATILE STRICT; diff --git a/test/support/CDB_UserTables.sql b/test/support/CDB_UserTables.sql new file mode 100644 index 00000000..a14c9002 --- /dev/null +++ b/test/support/CDB_UserTables.sql @@ -0,0 +1,28 @@ +-- Function returning list of cartodb user tables +-- +-- The optional argument restricts the result to tables +-- of the specified access type. +-- +-- Currently accepted permissions are: 'public', 'private' or 'all' +-- +DROP FUNCTION IF EXISTS CDB_UserTables(text); +CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all') +RETURNS SETOF name +AS $$ + +SELECT c.relname +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind = 'r' +AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys') +AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb') +AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT') + WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT') + WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT') + ELSE false END; + +$$ LANGUAGE 'sql'; + +-- This is to migrate from pre-0.2.0 version +-- See http://github.com/CartoDB/cartodb-postgresql/issues/36 +GRANT EXECUTE ON FUNCTION CDB_UserTables(text) TO public; diff --git a/test/support/CDB_ZoomFromScale.sql b/test/support/CDB_ZoomFromScale.sql new file mode 100644 index 00000000..6ded82e0 --- /dev/null +++ b/test/support/CDB_ZoomFromScale.sql @@ -0,0 +1,30 @@ +CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$ +BEGIN + CASE + WHEN scaleDenominator > 500000000 THEN RETURN 0; + WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1; + WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2; + WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3; + WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4; + WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5; + WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6; + WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7; + WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8; + WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9; + WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10; + WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11; + WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12; + WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13; + WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14; + WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15; + WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16; + WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17; + WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18; + WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19; + WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20; + WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21; + WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22; + WHEN scaleDenominator <= 100 THEN RETURN 23; + END CASE; +END +$$ LANGUAGE plpgsql IMMUTABLE;