From 24cb6cf9c150150e256a92ea9f59a7d9a2407195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 23 Dec 2019 20:13:11 +0100 Subject: [PATCH 1/6] Revert 0.33.0 --- NEWS.md | 3 + scripts-available/CDB_FederatedServer.sql | 452 +----------------- .../CDB_FederatedServerDiagnostics.sql | 252 +--------- .../CDB_FederatedServerListRemote.sql | 305 +----------- .../CDB_FederatedServerTables.sql | 354 +------------- scripts-available/CDB_ForeignTable.sql | 179 ++++++- scripts-available/CDB_Groups_API.sql | 1 + scripts-available/CDB_Organizations.sql | 26 +- sql/test_setup.sql | 2 +- test/CDB_CartodbfyTableTest.sql | 4 +- test/CDB_CartodbfyTableTest_expect | 4 +- test/CDB_FederatedServer.sql | 220 --------- test/CDB_FederatedServerDiagnostics.sql | 125 ----- test/CDB_FederatedServerDiagnostics_expect | 28 -- test/CDB_FederatedServerListRemote.sql | 319 ------------ test/CDB_FederatedServerListRemote_expect | 162 ------- test/CDB_FederatedServerTables.sql | 405 ---------------- test/CDB_FederatedServerTables_expect | 116 ----- test/CDB_FederatedServer_expect | 69 --- test/CDB_SyncTableTest.sql | 2 +- test/CDB_SyncTableTest_expect | 8 +- test/CDB_Username_expect | 2 +- test/extension/test.sh | 62 +++ 23 files changed, 310 insertions(+), 2790 deletions(-) delete mode 100644 test/CDB_FederatedServer.sql delete mode 100644 test/CDB_FederatedServerDiagnostics.sql delete mode 100644 test/CDB_FederatedServerDiagnostics_expect delete mode 100644 test/CDB_FederatedServerListRemote.sql delete mode 100644 test/CDB_FederatedServerListRemote_expect delete mode 100644 test/CDB_FederatedServerTables.sql delete mode 100644 test/CDB_FederatedServerTables_expect delete mode 100644 test/CDB_FederatedServer_expect diff --git a/NEWS.md b/NEWS.md index ae3b93f..a1f5890 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +0.34.0 (2019-12-23) +* Revert changes done in 0.32.0, keeping function signature to drop them + 0.33.0 (2019-12-20) * Revert `Make PG12 depend on plpython3u instead of plpythonu`. * Add functions to manage Federated Tables (Foreign Data Wrapper) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index fabdc62..a7885a5 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -1,437 +1,15 @@ --------------------------------------------------------------------------------- --- Private functions --------------------------------------------------------------------------------- - --- --- This function is just a placement to store and use the pattern for --- foreign object names --- Servers: cdb_fs_$(server_name) --- View schema: cdb_fs_$(server_name) --- > This is where all views created when importing tables are placed --- > One server has only one view schema --- Import Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) --- > This is where the foreign tables are placed --- > One server has one import schema per remote schema plus auxiliar ones used --- to access the remote catalog (pg_catalog, information_schema...) --- Owner role: cdb_fs_$(md5sum(current_database() || server_name) --- > This is the role than owns all schemas and tables related to the server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() -RETURNS TEXT -AS $$ - SELECT 'cdb_fs_'::text; -$$ -LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - --- --- Produce a valid DB name for servers generated for the Federated Server --- If check_existence is true, it'll throw if the server doesn't exists --- This name is also used as the schema to store views --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) -RETURNS NAME -AS $$ -DECLARE - internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); -BEGIN - IF input_name IS NULL OR char_length(input_name) = 0 THEN - RAISE EXCEPTION 'Server name cannot be NULL'; - END IF; - - -- We discard anything that would be truncated - IF (char_length(internal_server_name) >= 64) THEN - RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name; - END IF; - - IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN - RAISE EXCEPTION 'Server "%" does not exist', input_name; - END IF; - - RETURN internal_server_name::name; -END -$$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - --- --- Given the internal name for a remote server, it returns the name used by the user --- Reverses __CDB_FS_Generate_Server_Name --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME) -RETURNS TEXT -AS $$ - SELECT right(internal_server_name, - char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; -$$ -LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - --- --- Produce a valid name for a schema generated for the Federated Server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT) -RETURNS NAME -AS $$ -DECLARE - hash_value text := md5(internal_server_name::text || '__' || schema_name::text); -BEGIN - IF schema_name IS NULL THEN - RAISE EXCEPTION 'Schema name cannot be NULL'; - END IF; - RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name; -END -$$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - --- --- Produce a valid name for a role generated for the Federated Server --- This needs to include the current database in its hash to avoid collisions in clusters with more than one database --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME) -RETURNS NAME -AS $$ -DECLARE - hash_value text := md5(current_database()::text || '__' || internal_server_name::text); - role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value); -BEGIN - RETURN role_name::name; -END -$$ -LANGUAGE PLPGSQL STABLE PARALLEL SAFE; - --- --- Creates (if not exist) a schema to place the objects for a remote schema --- The schema is with the same AUTHORIZATION as the server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT) -RETURNS NAME -AS $$ -DECLARE - schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); -BEGIN - -- By changing the local role to the owner of the server we have an - -- easy way to check for permissions and keep all objects under the same owner - BEGIN - EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); - EXCEPTION - WHEN invalid_parameter_value THEN - RAISE EXCEPTION 'Server "%" does not exist', - @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); - WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to access the server "%"', - @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); - END; - - IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN - EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); - END IF; - RETURN schema_name; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- Returns the type of a server by internal name --- Currently all of them should be postgres_fdw --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME) -RETURNS name -AS $$ - SELECT f.fdwname - FROM pg_foreign_server s - JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid - WHERE s.srvname = internal_server_name; -$$ -LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - --- --- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB) -RETURNS jsonb -AS $$ -DECLARE - mapping jsonb := '{}'::jsonb; -BEGIN - IF NOT (input_config ? 'credentials') THEN - RAISE EXCEPTION 'Credentials are mandatory'; - END IF; - - -- For now, allow not passing username or password - IF input_config->'credentials'->'username' IS NOT NULL THEN - mapping := jsonb_build_object('user', input_config->'credentials'->'username'); - END IF; - IF input_config->'credentials'->'password' IS NOT NULL THEN - mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password'); - END IF; - - RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping); -END -$$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - --- Take a config jsonb as input and return it augmented with default --- options -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb) -RETURNS jsonb -AS $$ -DECLARE - default_options jsonb := '{ - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - }'; - server_config jsonb; -BEGIN - IF NOT (input_config ? 'server') THEN - RAISE EXCEPTION 'Server information is mandatory'; - END IF; - server_config := default_options || to_jsonb(input_config->'server'); - RETURN jsonb_set(input_config, '{server}'::text[], server_config); -END -$$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - --- Given an server name, returns the username used in the configuration if the caller has rights to access it -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME) -RETURNS text -AS $$ -DECLARE - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); - username text; -BEGIN - BEGIN - EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); - EXCEPTION WHEN OTHERS THEN - RETURN NULL; - END; - - SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username - FROM pg_foreign_server s - LEFT JOIN pg_user_mappings u - ON u.srvid = s.oid - WHERE s.srvname = internal_server_name - ORDER BY 1; - - RESET ROLE; - - RETURN username; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --------------------------------------------------------------------------------- --- Public functions --------------------------------------------------------------------------------- - - --- --- Registers a new PG server --- --- Example config: '{ --- "server": { --- "dbname": "fdw_target", --- "host": "localhost", --- "port": 5432, --- "extensions": "postgis", --- "updatable": "false", --- "use_remote_estimate": "true", --- "fetch_size": "1000" --- }, --- "credentials": { --- "username": "fdw_user", --- "password": "foobarino" --- } --- }' --- --- The configuration from __CDB_FS_add_default_options will be appended --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); - final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); - row record; - option record; -BEGIN - IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN - RAISE EXCEPTION 'postgres_fdw extension is not installed' - USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`'; - END IF; - - -- We only create server and roles if the server didn't exist before - IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN - BEGIN - EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN - EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); - END IF; - EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); - - -- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases - -- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database - EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name); - EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name); - EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name); - EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name); - EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name); - - EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); - EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); - EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); - EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); - EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM - USING HINT = 'Please clean the left over objects'; - END; - END IF; - - -- Add new options - FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p - LOOP - IF NOT EXISTS ( - WITH a AS ( - SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal - ) SELECT * from a where options = row.key) - THEN - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value); - ELSE - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value); - END IF; - END LOOP; - - -- Update user mapping settings - FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o - LOOP - IF NOT EXISTS ( - WITH a AS ( - SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name - ) SELECT * from a where options = option.key) - THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value); - ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value); - END IF; - END LOOP; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- Drops a registered server and all the objects associated with it --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); -BEGIN - SET client_min_messages = ERROR; - BEGIN - EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); - EXECUTE FORMAT ('DROP OWNED BY %I', role_name); - EXECUTE FORMAT ('DROP ROLE %I', role_name); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; - END; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List registered servers --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') -RETURNS TABLE ( - name text, - driver text, - host text, - port text, - dbname text, - readmode text, - username text -) -AS $$ -DECLARE - server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server); -BEGIN - RETURN QUERY SELECT - -- Name as shown to the user - @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", - - -- Which driver are we using (postgres_fdw, odbc_fdw...) - @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver", - - -- Read options from pg_foreign_server - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", - CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", - - @extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username" - FROM pg_foreign_server s - LEFT JOIN pg_user_mappings u - ON u.srvid = s.oid - WHERE s.srvname ILIKE server_name - ORDER BY 1; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE; - - --- --- Grant access to a server --- In the future we might consider adding the server's view schema to the role search_path --- to make it easier to access the created views --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); -BEGIN - IF (db_role IS NULL) THEN - RAISE EXCEPTION 'User role "%" cannot be NULL', username; - END IF; - BEGIN - EXECUTE format('GRANT %I TO %I', server_role_name, db_role); - EXCEPTION - WHEN insufficient_privilege THEN - RAISE EXCEPTION 'You do not have rights to grant access on "%"', server; - WHEN OTHERS THEN - RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM; - END; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- Revoke access to a server --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); -BEGIN - IF (db_role IS NULL) THEN - RAISE EXCEPTION 'User role "%" cannot be NULL', username; - END IF; - BEGIN - EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role); - EXCEPTION - WHEN insufficient_privilege THEN - RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server; - WHEN OTHERS THEN - RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM; - END; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Name_Pattern(); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Name(TEXT, BOOL); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Extract_Server_Name(NAME); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Schema_Name(NAME, TEXT); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Role_Name(NAME); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Create_Schema(NAME, TEXT); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_server_type(NAME); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_credentials_to_user_mapping(JSONB); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_add_default_options(jsonb); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_get_usermapping_username(NAME); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Register_PG(TEXT, JSONB); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Unregister(TEXT); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Servers(TEXT); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Grant_Access(TEXT, NAME); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Revoke_Access(TEXT, NAME); diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index b1e2e4c..b05b6d3 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -1,243 +1,9 @@ --------------------------------------------------------------------------------- --- Private functions --------------------------------------------------------------------------------- - --- --- Import a foreign table if it does not exist --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name) -RETURNS void -AS $$ -DECLARE - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); -BEGIN - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_table - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', - remote_schema, remote_table, server_internal, local_schema); - END IF; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- Get the version of a remote PG server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name) -RETURNS text -AS $$ -DECLARE - remote_schema name := 'pg_catalog'; - remote_table name := 'pg_settings'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - remote_server_version text; -BEGIN - PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); - - BEGIN - EXECUTE format(' - SELECT setting FROM %I.%I WHERE name = ''server_version''; - ', local_schema, remote_table) INTO remote_server_version; - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to access the server "%"', - @extschema@.__CDB_FS_Extract_Server_Name(server_internal); - END; - - RETURN remote_server_version; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --- --- Get the PostGIS extension version of a remote PG server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name) -RETURNS text -AS $$ -DECLARE - remote_schema name := 'pg_catalog'; - remote_table name := 'pg_extension'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - remote_postgis_version text; -BEGIN - PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); - - BEGIN - EXECUTE format(' - SELECT extversion FROM %I.%I WHERE extname = ''postgis''; - ', local_schema, remote_table) INTO remote_postgis_version; - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to access the server "%"', - @extschema@.__CDB_FS_Extract_Server_Name(server_internal); - END; - - RETURN remote_postgis_version; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --- --- Get the foreign server options of a remote PG server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name) -RETURNS jsonb -AS $$ - -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html - -- See https://www.postgresql.org/docs/current/functions-info.html - SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM ( - SELECT (pg_options_to_table(srvoptions)).* - FROM pg_foreign_server - WHERE srvname = server_internal - ) AS opt; -$$ -LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - - --- --- Get a foreign PG server hostname from the catalog --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name) -RETURNS text -AS $$ - SELECT option_value FROM ( - SELECT (pg_options_to_table(srvoptions)).* - FROM pg_foreign_server WHERE srvname = server_internal - ) AS opt WHERE opt.option_name = 'host'; -$$ -LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - - --- --- Get a foreign PG server port from the catalog --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name) -RETURNS integer -AS $$ - SELECT option_value::integer FROM ( - SELECT (pg_options_to_table(srvoptions)).* - FROM pg_foreign_server WHERE srvname = server_internal - ) AS opt WHERE opt.option_name = 'port'; -$$ -LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - --- --- Get one measure of network latency in ms to a remote TCP server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency( - server_internal name, - timeout_seconds float DEFAULT 5.0, - n_samples integer DEFAULT 10 -) -RETURNS jsonb -AS $$ - import socket - import json - import math - from timeit import default_timer as timer - - plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name']) - rv = plpy.execute(plan, [server_internal], 1) - host = rv[0]['host'] - - plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name']) - rv = plpy.execute(plan, [server_internal], 1) - port = rv[0]['port'] or 5432 - - n_errors = 0 - samples = [] - - for i in range(n_samples): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout_seconds) - - t_start = timer() - - try: - s.connect((host, int(port))) - t_stop = timer() - s.shutdown(socket.SHUT_RD) - except (socket.timeout, OSError, socket.error) as ex: - plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) - n_errors += 1 - finally: - s.close() - - t_connect = (t_stop - t_start) * 1000.0 - plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) - samples.append(t_connect) - - stats = { - 'n_samples': n_samples, - 'n_errors': n_errors, - } - n = len(samples) - if n >= 1: - mean = sum(samples) / n - stats.update({ - 'avg': round(mean, 3), - 'min': round(min(samples), 3), - 'max': round(max(samples), 3) - }) - if n >= 2: - var = sum([ (x - mean)**2 for x in samples ]) / (n-1) - stdev = math.sqrt(var) - stats.update({ - 'stdev': round(stdev, 3) - }) - return json.dumps(stats) -$$ -LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; - - --- --- Collect and return diagnostics info from a remote PG into a jsonb --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name) -RETURNS jsonb -AS $$ -DECLARE - remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); - remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); - remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); - remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal); -BEGIN - RETURN jsonb_build_object( - 'server_version', remote_server_version, - 'postgis_version', remote_postgis_version, - 'server_options', remote_server_options, - 'server_latency_ms', remote_server_latency_ms - ); -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - - --------------------------------------------------------------------------------- --- Public functions --------------------------------------------------------------------------------- - --- --- Collect and return diagnostics info from a remote PG into a jsonb --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT) -RETURNS jsonb -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_type name := @extschema@.__CDB_FS_server_type(server_internal); -BEGIN - CASE server_type - WHEN 'postgres_fdw' THEN - RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal); - ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; - END CASE; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Import_If_Not_Exists(name, name, name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Version_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Options_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Host_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Port_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(name, float, integer); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Server_Diagnostics_PG(name); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Diagnostics(TEXT); diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 0367027..1632252 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -1,298 +1,7 @@ --------------------------------------------------------------------------------- --- Private functions --------------------------------------------------------------------------------- - --- --- List the schemas of a remote PG server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name) -RETURNS TABLE(remote_schema name) -AS $$ -DECLARE - -- Import schemata from the information schema - -- - -- "The view schemata contains all schemas in the current database - -- that the current user has access to (by way of being the owner - -- or having some privilege)." - -- See https://www.postgresql.org/docs/11/infoschema-schemata.html - -- - -- "The information schema is defined in the SQL standard and can - -- therefore be expected to be portable and remain stable" - -- See https://www.postgresql.org/docs/11/information-schema.html - inf_schema name := 'information_schema'; - remote_table name := 'schemata'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); -BEGIN - -- Import the foreign schemata table - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_table - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', - inf_schema, remote_table, server_internal, local_schema); - END IF; - - -- Return the result we're interested in. Exclude toast and temp schemas - BEGIN - RETURN QUERY EXECUTE format(' - SELECT schema_name::name AS remote_schema FROM %I.%I - WHERE schema_name NOT LIKE %s - ORDER BY remote_schema - ', local_schema, remote_table, '''pg_%'''); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to access the server "%"', - @extschema@.__CDB_FS_Extract_Server_Name(server_internal); - END; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List the names of the tables in a remote PG schema --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name) -RETURNS TABLE(remote_table name) -AS $func$ -DECLARE - -- Import `tables` from the information schema - -- - -- "The view tables contains all tables and views defined in the - -- current database. Only those tables and views are shown that - -- the current user has access to (by way of being the owner or - -- having some privilege)." - -- https://www.postgresql.org/docs/11/infoschema-tables.html - - -- Create local target schema if it does not exists - inf_schema name := 'information_schema'; - remote_table name := 'tables'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); -BEGIN - -- Import the foreign `tables` if not done - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_table - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', - inf_schema, remote_table, server_internal, local_schema); - END IF; - - -- Note: in this context, schema names are not to be quoted - RETURN QUERY EXECUTE format($q$ - SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name - $q$, local_schema, remote_table, remote_schema); -END -$func$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --- --- List the columns in a remote PG schema --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name) -RETURNS TABLE(table_name name, column_name name, column_type text) -AS $func$ -DECLARE - -- Import `columns` from the information schema - -- - -- "The view columns contains information about all table columns (or view columns) - -- in the database. System columns (oid, etc.) are not included. Only those columns - -- are shown that the current user has access to (by way of being the owner or having some privilege)." - -- https://www.postgresql.org/docs/11/infoschema-columns.html - - -- Create local target schema if it does not exists - inf_schema name := 'information_schema'; - remote_col_table name := 'columns'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); -BEGIN - -- Import the foreign `columns` if not done - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_col_table - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', - inf_schema, remote_col_table, server_internal, local_schema); - END IF; - - -- Note: in this context, schema names are not to be quoted - -- We join with the geometry columns to change the type `USER-DEFINED` - -- by its appropiate geometry and srid - RETURN QUERY EXECUTE format($q$ - SELECT - a.table_name::name, - a.column_name::name, - COALESCE(b.column_type, a.data_type)::TEXT as column_type - FROM - %I.%I a - LEFT JOIN - @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b - ON a.table_name = b.table_name AND a.column_name = b.column_name - WHERE table_schema = '%s' - ORDER BY a.table_name, a.column_name $q$, - local_schema, remote_col_table, - server_internal, remote_schema, - remote_schema); -END -$func$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List the geometry columns in a remote PG schema --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public') -RETURNS TABLE(table_name name, column_name name, column_type text) -AS $func$ -DECLARE - -- Import `geometry_columns` and `geography_columns` from the postgis schema - -- We assume that postgis is installed in the public schema - - -- Create local target schema if it does not exists - remote_geometry_view name := 'geometry_columns'; - remote_geography_view name := 'geography_columns'; - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema); -BEGIN - -- Import the foreign `geometry_columns` and `geography_columns` if not done - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_geometry_view - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I', - postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema); - END IF; - - BEGIN - -- Note: We return both the type and srid as the type - RETURN QUERY EXECUTE format($q$ - SELECT - f_table_name::NAME as table_name, - f_geometry_column::NAME as column_name, - type::TEXT || ',' || srid::TEXT as column_type - FROM - ( - SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I - ) _geo_views - WHERE f_table_schema = '%s' - $q$, - local_schema, remote_geometry_view, - local_schema, remote_geography_view, - remote_schema); - EXCEPTION WHEN OTHERS THEN - RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"', - postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal); - RETURN; - END; -END -$func$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --------------------------------------------------------------------------------- --- Public functions --------------------------------------------------------------------------------- - --- --- List remote schemas in a federated server that the current user has access to. --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT) -RETURNS TABLE(remote_schema name) -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_type name := @extschema@.__CDB_FS_server_type(server_internal); -BEGIN - CASE server_type - WHEN 'postgres_fdw' THEN - RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal); - ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; - END CASE; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List remote tables in a federated server that the current user has access to. --- For registered tables it returns also the associated configuration --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT) -RETURNS TABLE( - registered boolean, - remote_table TEXT, - local_qualified_name TEXT, - id_column_name TEXT, - geom_column_name TEXT, - webmercator_column_name TEXT, - columns JSON - ) -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_type name := @extschema@.__CDB_FS_server_type(server_internal); -BEGIN - CASE server_type - WHEN 'postgres_fdw' THEN - RETURN QUERY - SELECT - coalesce(registered_tables.registered, false)::boolean as registered, - foreign_tables.remote_table::text as remote_table, - registered_tables.local_qualified_name as local_qualified_name, - registered_tables.id_column_name as id_column_name, - registered_tables.geom_column_name as geom_column_name, - registered_tables.webmercator_column_name as webmercator_column_name, - remote_columns.columns as columns - FROM - @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables - LEFT JOIN - @extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables - ON foreign_tables.remote_table = registered_tables.remote_table - LEFT JOIN - ( -- Extract and group columns with their remote table - SELECT table_name, - json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns - FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) - GROUP BY table_name - ) remote_columns - ON foreign_tables.remote_table = remote_columns.table_name - ORDER BY foreign_tables.remote_table; - ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; - END CASE; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List the columns of a remote table in a federated server that the current user has access to. --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( - server TEXT, - remote_schema TEXT, - remote_table TEXT) -RETURNS TABLE(column_n name, column_t text) -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); - server_type name := @extschema@.__CDB_FS_server_type(server_internal); -BEGIN - IF remote_table IS NULL THEN - RAISE EXCEPTION 'Remote table name cannot be NULL'; - END IF; - - CASE server_type - WHEN 'postgres_fdw' THEN - RETURN QUERY - SELECT - column_name, - column_type - FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) - WHERE table_name = remote_table - ORDER BY column_name; - ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; - END CASE; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Schemas_PG(name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Tables_PG(name, name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Columns_PG(name, name); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(name, name, name); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Schemas(TEXT); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Tables(TEXT, TEXT); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Columns(TEXT, TEXT, TEXT); diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 302b3bd..5149644 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -1,345 +1,9 @@ --------------------------------------------------------------------------------- --- Private functions --------------------------------------------------------------------------------- - --- --- Checks if a column is of integer type --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME) -RETURNS boolean -AS $$ -BEGIN - PERFORM atttypid FROM pg_catalog.pg_attribute - WHERE attrelid = input_table - AND attname = colname - AND atttypid IN (SELECT oid FROM pg_type - WHERE typname IN - ('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8')); - RETURN FOUND; -END -$$ -LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; - --- --- Checks if a column is of geometry type --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME) -RETURNS boolean -AS $$ -BEGIN - PERFORM atttypid FROM pg_catalog.pg_attribute - WHERE attrelid = input_table - AND attname = colname - AND atttypid = '@postgisschema@.geometry'::regtype; - RETURN FOUND; -END -$$ -LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; - --- --- Returns the name of all the columns from a table --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS) -RETURNS SETOF NAME -AS $$ - SELECT - a.attname as "colname" - FROM pg_catalog.pg_attribute a - WHERE - a.attnum > 0 - AND NOT a.attisdropped - AND a.attrelid = ( - SELECT c.oid - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE c.oid = input_table::oid - ) - ORDER BY a.attnum; -$$ LANGUAGE SQL STABLE PARALLEL UNSAFE; - --- --- Returns the id column from a view definition --- Note: The id is always of one of this forms: --- SELECT t.cartodb_id, --- SELECT t.my_id as cartodb_id, --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT) -RETURNS TEXT -AS $$ - WITH column_definitions AS - ( - SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def - ) - SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2)) - FROM column_definitions - WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id') - LIMIT 1; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - --- --- Returns the geom column from a view definition --- --- Note: The the_geom is always of one of this forms: --- t.the_geom, --- t.my_geom as the_geom, --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT) -RETURNS TEXT -AS $$ - WITH column_definitions AS - ( - SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def - ) - SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) - FROM column_definitions - WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom') - LIMIT 1; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - --- --- Returns the webmercatorcolumn from a view definition --- Note: The the_geom_webmercator is always of one of this forms: --- t.the_geom_webmercator, --- t.my_geom as the_geom_webmercator, --- Or without the trailing comma: --- t.the_geom_webmercator --- t.my_geom as the_geom_webmercator --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT) -RETURNS TEXT -AS $$ - WITH column_definitions AS - ( - SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def - ) - SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) - FROM column_definitions - WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator') - LIMIT 1; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - - --- --- List all registered tables in a server + schema --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables( - server_internal NAME, - remote_schema TEXT - ) -RETURNS TABLE( - registered boolean, - remote_table TEXT, - local_qualified_name TEXT, - id_column_name TEXT, - geom_column_name TEXT, - webmercator_column_name TEXT - ) -AS $$ -DECLARE - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); -BEGIN - RETURN QUERY SELECT - true as registered, - source_table::text as remote_table, - format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name, - @extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name, - @extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name, - @extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name - FROM - ( - SELECT DISTINCT - dependent_ns.nspname as dependent_schema, - dependent_view.relname as dependent_view, - source_table.relname as source_table, - pg_get_viewdef(dependent_view.oid) as view_definition - FROM pg_depend - JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid - JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid - JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid - JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace - JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace - WHERE - source_ns.nspname = local_schema - ORDER BY 1,2 - ) _aux; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - --------------------------------------------------------------------------------- --- Public functions --------------------------------------------------------------------------------- - --- --- Sets up a Federated Table --- --- Precondition: the federated server has to be set up via --- CDB_Federated_Server_Register_PG --- --- Postcondition: it generates a view in the schema of the user that --- can be used through SQL and Maps API's. --- If the table was already exported, it will be dropped and re-imported --- --- The view is placed under the server's view schema (cdb_fs_$(server_name)) --- --- E.g: --- SELECT cartodb.CDB_SetUp_PG_Federated_Table( --- 'amazon', -- mandatory, name of the federated server --- 'my_remote_schema', -- mandatory, schema name --- 'my_remote_table', -- mandatory, table name --- 'id', -- mandatory, name of the id column --- 'geom', -- optional, name of the geom column, preferably in 4326 --- 'webmercator' -- optional, should be in 3857 if present --- 'local_name' -- optional, name of the local view (uses the remote_name if not declared) --- ); --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( - server TEXT, - remote_schema TEXT, - remote_table TEXT, - id_column TEXT, - geom_column TEXT DEFAULT NULL, - webmercator_column TEXT DEFAULT NULL, - local_name NAME DEFAULT NULL -) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); - - src_table REGCLASS; -- import_schema.remote_table - import_schema is local_schema - local_view text; -- view_schema.local_name - view_schema is server_internal - - rest_of_cols TEXT[]; - geom_expression TEXT; - webmercator_expression TEXT; - carto_columns_expression TEXT[]; -BEGIN - IF remote_table IS NULL THEN - RAISE EXCEPTION 'Remote table name cannot be NULL'; - END IF; - - -- Make do with whatever columns are provided - IF webmercator_column IS NULL THEN - webmercator_column := geom_column; - ELSIF geom_column IS NULL THEN - geom_column := webmercator_column; - END IF; - - IF local_name IS NULL THEN - local_name := remote_table; - END IF; - - -- Import the foreign table - -- Drop the old view / table if there was one - IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN - EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table); - END IF; - BEGIN - EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', - remote_schema, remote_table, server_internal, local_schema); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM; - END; - - BEGIN - src_table := format('%I.%I', local_schema, remote_table); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server; - END; - - -- Check id_column is numeric - IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN - RAISE EXCEPTION 'non integer id_column "%"', id_column; - END IF; - - -- Check if the geom and mercator columns have a geometry type (if provided) - IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN - RAISE EXCEPTION 'non geometry column "%"', geom_column; - END IF; - IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN - RAISE EXCEPTION 'non geometry column "%"', webmercator_column; - END IF; - - -- Get a list of columns excluding the id, geom and the_geom_webmercator - SELECT ARRAY( - SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c - WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL) - ) INTO rest_of_cols; - - IF geom_column IS NULL - THEN - geom_expression := 'NULL AS the_geom'; - ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326 - THEN - geom_expression := format('t.%I AS the_geom', geom_column); - ELSE - -- It needs an ST_Transform to 4326 - geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column); - END IF; - - IF webmercator_column IS NULL - THEN - webmercator_expression := 'NULL AS the_geom_webmercator'; - ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857 - THEN - webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column); - ELSE - -- It needs an ST_Transform to 3857 - webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column); - END IF; - - -- CARTO columns expressions - carto_columns_expression := ARRAY[ - format('t.%1$I AS cartodb_id', id_column), - geom_expression, - webmercator_expression - ]; - - -- Create view schema if it doesn't exist - IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN - EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name); - END IF; - - -- Create a view with homogeneous CDB fields - BEGIN - EXECUTE format( - 'CREATE OR REPLACE VIEW %1$I.%2$I AS - SELECT %3s - FROM %4$s t', - server_internal, local_name, - array_to_string(carto_columns_expression || rest_of_cols, ','), - src_table - ); - EXCEPTION WHEN OTHERS THEN - IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN - RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name; - ELSE - RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; - END IF; - END; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- Unregisters a remote table. Any dependent object will be dropped --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister( - server TEXT, - remote_schema TEXT, - remote_table TEXT -) -RETURNS void -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); -BEGIN - EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table); -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Integer(REGCLASS, NAME); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Geometry(REGCLASS, NAME); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_GetColumns(REGCLASS); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_id_column(TEXT); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_geom_column(TEXT); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_webmercator_column(TEXT); +DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Registered_Tables(NAME,TEXT); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Register(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, NAME); +DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Unregister(TEXT, TEXT, TEXT); diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index b9449de..c86f5ef 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -139,6 +139,176 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; +-- Produce a valid DB name for objects created for the user FDW's +CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME) +RETURNS NAME AS $$ + -- Note on input we use %s and on output we use %I, in order to + -- avoid double escaping + SELECT format('cdb_fdw_%s', fdw_input_name)::NAME; +$$ +LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +-- A function to set up a user-defined foreign data server +-- It does not read from CDB_Conf. +-- Only superuser roles can invoke it successfully +-- +-- Sample call: +-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{ +-- "server": { +-- "extensions": "postgis", +-- "dbname": "testdb", +-- "host": "myhostname.us-east-2.rds.amazonaws.com", +-- "port": "5432" +-- }, +-- "user_mapping": { +-- "user": "fdw_user", +-- "password": "secret" +-- } +-- }'); +-- +-- Underneath it will: +-- * Set up postgresql_fdw +-- * Create a server with the name 'cdb_fdw_amazon' +-- * Create a role called 'cdb_fdw_amazon' to manage access +-- * Create a user mapping with that role 'cdb_fdw_amazon' +-- * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign +-- tables over there +-- +-- It is the responsibility of the superuser to grant that role to either: +-- * Nobody +-- * Specific roles: GRANT amazon TO role_name; +-- * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon'); +-- * The publicuser: GRANT cdb_fdw_amazon TO publicuser; +CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json) +RETURNS void AS $$ +DECLARE + row record; + option record; + fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); +BEGIN + -- TODO: refactor with original function + -- This function tries to be as idempotent as possible, by not creating anything more than once + -- (not even using IF NOT EXIST to avoid throwing warnings) + IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN + CREATE EXTENSION postgres_fdw; + RAISE NOTICE 'Created postgres_fdw EXTENSION'; + END IF; + -- Create FDW first if it does not exist + IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name) + THEN + EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name); + RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name; + END IF; + + -- Set FDW settings + FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p + LOOP + IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key) + THEN + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value); + ELSE + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value); + END IF; + END LOOP; + + -- Create specific role for this + IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN + EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name); + RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name; + END IF; + + -- Transfer ownership of the server to the fdw role + EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name); + + -- Create user mapping + -- NOTE: we use a PUBLIC user mapping but control access to the SERVER + -- so that we don't need to create a mapping for every user nor store credentials elsewhere + IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN + EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name); + RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name; + END IF; + + -- Update user mapping settings + FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP + IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value); + ELSE + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value); + END IF; + END LOOP; + + -- Grant usage on the wrapper and server to the fdw role + EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name); + RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name; + EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name); + RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name; + + -- Create schema if it does not exist. + IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN + EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name); + RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name; + END IF; + + -- Give the fdw role ownership over the schema + EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name); + RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name; + + -- TODO: Bring here the remote cdb_tablemetadata +END +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; + + +-- A function to drop a user-defined foreign server and all related objects +-- It does not read from CDB_Conf +-- It must be executed with a superuser role to succeed +-- +-- Sample call: +-- SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon') +-- +-- Note: if there's any dependent object (i.e. foreign table) this call will fail +CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false) +RETURNS void AS $$ +DECLARE + fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); + cascade_clause NAME; +BEGIN + CASE force + WHEN true THEN + cascade_clause := 'CASCADE'; + ELSE + cascade_clause := 'RESTRICT'; + END CASE; + + EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause); + RAISE NOTICE 'Dropped schema %', fdw_objects_name; + EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name); + RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name; + EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause); + RAISE NOTICE 'Dropped foreign server %', fdw_objects_name; + EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause); + RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name; + EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name); + RAISE NOTICE 'Dropped role %', fdw_objects_name; +END +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; + + +-- Set up a user foreign table +-- E.g: +-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable'); +-- SELECT * FROM amazon.my_table; +CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME) +RETURNS void AS $$ +DECLARE + fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); +BEGIN + EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name); + --- Grant SELECT to fdw role + EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); +END +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; + + CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid) RETURNS TEXT AS $$ SELECT option_value FROM pg_options_to_table(( @@ -204,12 +374,3 @@ RETURNS timestamptz AS $$ LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid ) SELECT max(updated_at) FROM t_updated_at; $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - - --------------------------------------------------------------------------------- --- Deprecated --------------------------------------------------------------------------------- -DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name); -DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json); -DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean); -DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name); diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index b29eb77..660d4e8 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -5,6 +5,7 @@ -- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }'); ---------------------------------- +-- TODO: delete this development cleanup before final merge DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text); diff --git a/scripts-available/CDB_Organizations.sql b/scripts-available/CDB_Organizations.sql index 94e228e..c532ed5 100644 --- a/scripts-available/CDB_Organizations.sql +++ b/scripts-available/CDB_Organizations.sql @@ -172,7 +172,27 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -------------------------------------------------------------------------------- --- Deprecated +-- Role management -------------------------------------------------------------------------------- -DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name); -DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name); +CREATE OR REPLACE +FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name) +RETURNS VOID AS $$ +DECLARE + org_role TEXT; +BEGIN + org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); + EXECUTE format('GRANT %I TO %I', role_name, org_role); +END +$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE +FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name) +RETURNS VOID AS $$ +DECLARE + org_role TEXT; +BEGIN + org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); + EXECUTE format('REVOKE %I FROM %I', role_name, org_role); +END +$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/sql/test_setup.sql b/sql/test_setup.sql index cf55bf4..ec5021b 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -2,7 +2,7 @@ \set QUIET on SET client_min_messages TO error; CREATE EXTENSION postgis; -CREATE EXTENSION @@plpythonu@@; +CREATE EXTENSION plpythonu; CREATE SCHEMA cartodb; \i 'cartodb--unpackaged--@@VERSION@@.sql' CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 1963eaf..3e5658e 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -134,9 +134,9 @@ DROP TABLE t; -- table with single non-geometrical column CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT cartodb_id, the_geom, description FROM t; +SELECT * FROM t; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT cartodb_id, the_geom, description FROM t; +SELECT * FROM t; DROP TABLE t; -- table with existing srid-unconstrained (but type-constrained) the_geom diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index c5030a0..9939598 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -7,9 +7,9 @@ single non-geometrical column cartodbfied fine DROP TABLE SELECT 1 check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence DROP TABLE SELECT 1 srid-unconstrained the_geom cartodbfied fine diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql deleted file mode 100644 index d314790..0000000 --- a/test/CDB_FederatedServer.sql +++ /dev/null @@ -1,220 +0,0 @@ --- Setup -\set QUIET on -SET client_min_messages TO error; -\set VERBOSITY terse -SET SESSION AUTHORIZATION postgres; -CREATE EXTENSION postgres_fdw; -\set QUIET off - -\echo '## List empty servers shows nothing' -SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## List non-existent server shows nothing' -SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server => 'doesNotExist'); - -\echo '## Create and list a server works' -SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "fdw_user", - "password": "foobarino" - } -}'::jsonb); -SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## Create and list a second server works' -SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ - "server": { - "dbname": "fdw_target", - "host": "localhost", - "port": @@PGPORT@@, - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - }, - "credentials": { - "username": "fdw_user", - "password": "foobarino" - } -}'::jsonb); -SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## List server by name works' -SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server => 'myRemote'); - - -\echo '## Re-register a second server works' -SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ - "server": { - "dbname": "fdw_target", - "host": "localhost", - "port": @@PGPORT@@, - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - }, - "credentials": { - "username": "other_remote_user", - "password": "foobarino" - } -}'::jsonb); -SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## Unregister server 1 works' -SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote'::text); -SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## Unregistering a server that does not exist fails' -SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server => 'doesNotExist'::text); - -\echo '## Unregister the second server works' -SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote2'::text); -SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## Create a server with NULL name fails' -SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server => NULL::text, config => '{ "server": {}, "credentials" : {}}'); -\echo '## Create a server with NULL config fails' -SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => NULL::jsonb); -\echo '## Create a server with empty config fails' -SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{}'); -\echo '## Create a server without credentials fails' -SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ - "server": { - "dbname": "fdw_target", - "host": "localhost", - "port": @@PGPORT@@, - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - } -}'::jsonb); -\echo '## Create a server with empty credentials works' -SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ - "server": { - "dbname": "fdw_target", - "host": "localhost", - "port": @@PGPORT@@, - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - }, - "credentials": { } -}'::jsonb); -SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); -SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server => 'empty'::text); -\echo '## Create a server without options fails' -SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ - "credentials": { - "username": "other_remote_user", - "password": "foobarino" - } -}'::jsonb); - -\echo '## Create a server with special characters works' -SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote" or''not'::text, config => '{ - "server": { - "dbname": "fdw target", - "host": "localhost", - "port": @@PGPORT@@, - "extensions": "postgis", - "updatable": "false", - "use_remote_estimate": "true", - "fetch_size": "1000" - }, - "credentials": { - "username": "fdw user", - "password": "foo barino" - } -}'::jsonb); -SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers(); -SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote" or''not'::text); - --- Test permissions -\set QUIET on -CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; -\set QUIET off - -SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote3'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "fdw_user", - "password": "foobarino" - } -}'::jsonb); - -\c contrib_regression cdb_fs_tester - -\echo '## All users are able to list servers' -SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); - -\echo '## Only superadmins can create servers' -SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "fdw_user", - "password": "foobarino" - } -}'::jsonb); - - -\c contrib_regression postgres - -\echo '## Granting access to a user works' -SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT '9.55', cartodb.CDB_Federated_Server_List_Servers(); -\c contrib_regression postgres -SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name); -SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name); - -\echo '## Granting access again raises a notice' -SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); - -\echo '## Revoking access to a user works' -SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); -SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); - -\echo '## Unregistering a server with active grants works' -SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote3'::text); - - -\echo '## A user with granted access can not drop a server' -SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "fdw_user", - "password": "foobarino" - } -}'::jsonb); -SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote4', db_role => 'cdb_fs_tester'::name); - -\c contrib_regression cdb_fs_tester -SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); - -\c contrib_regression postgres -SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); - - --- Cleanup -\set QUIET on -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; -DROP ROLE cdb_fs_tester; -DROP EXTENSION postgres_fdw; -\set QUIET off diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql deleted file mode 100644 index da5e8cb..0000000 --- a/test/CDB_FederatedServerDiagnostics.sql +++ /dev/null @@ -1,125 +0,0 @@ --- =================================================================== --- create FDW objects --- =================================================================== -\set QUIET on -SET client_min_messages TO error; -\set VERBOSITY terse -CREATE EXTENSION postgres_fdw; - -CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; - --- Create database to be used as remote -CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; - -SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - -SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ - "server": { - "host": "localhost", - "port": "12345" - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - -SELECT 'C3', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback-no-port'::text, config => '{ - "server": { - "host": "localhost" - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - -\c cdb_fs_tester postgres -CREATE EXTENSION postgis; -\c contrib_regression postgres -\set QUIET off - - --- =================================================================== --- Test server diagnostics function(s) --- =================================================================== -\echo '%% It raises an error if the server does not exist' -SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist'); - -\echo '%% It returns a jsonb object' -SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')); - -\echo '%% It returns the server version' -SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb - FROM pg_settings WHERE name = 'server_version'; - -\echo '%% It returns the postgis version' -SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb - FROM pg_extension WHERE extname = 'postgis'; - -\echo '%% It returns null as the postgis version if it is not installed' -\set QUIET on -\c cdb_fs_tester postgres -DROP EXTENSION postgis; -\c contrib_regression postgres -\set QUIET off -SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb; - -\echo '%% It returns the remote server options' -SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; - -\echo '%% It returns network latency stats to the remote server: min <= avg <= max' -WITH latency AS ( - SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms -) SELECT '2.1', (latency.ms->'min')::text::float <= (latency.ms->'avg')::text::float, (latency.ms->'avg')::text::float <= (latency.ms->'max')::text::float -FROM latency; - -\echo '%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)' -WITH latency AS ( - SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms -) SELECT '2.2', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 -FROM latency; - -\echo '%% Latency stats: stdev > 0' -WITH latency AS ( - SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms -) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0 -FROM latency; - -\echo '%% It raises an error if the wrong port is provided' -SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port'); - -\echo '%% Latency stats: can get them on default PG port 5432 when not provided' -WITH latency AS ( - SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms -) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 -FROM latency; - - --- =================================================================== --- Cleanup --- =================================================================== -\set QUIET on -SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); -SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text); -SELECT 'D3', cartodb.CDB_Federated_Server_Unregister(server => 'loopback-no-port'::text); --- Reconnect, using a new session in order to close FDW connections -\connect -DROP DATABASE cdb_fs_tester; - --- Drop role -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; -DROP ROLE cdb_fs_tester; - -DROP EXTENSION postgres_fdw; -\set QUIET off diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect deleted file mode 100644 index 044ad2f..0000000 --- a/test/CDB_FederatedServerDiagnostics_expect +++ /dev/null @@ -1,28 +0,0 @@ -C1| -C2| -C3| -%% It raises an error if the server does not exist -ERROR: Server "doesNotExist" does not exist -%% It returns a jsonb object -1.2|jsonb -%% It returns the server version -1.3|t -%% It returns the postgis version -1.4|t -%% It returns null as the postgis version if it is not installed -1.5|t -%% It returns the remote server options -1.6|t -%% It returns network latency stats to the remote server: min <= avg <= max -2.1|t|t -%% Latency stats: 0 <= min <= max <= 1000 ms (local connection) -2.2|t|t -%% Latency stats: stdev > 0 -2.3|t -%% It raises an error if the wrong port is provided -ERROR: could not connect to server "cdb_fs_wrong-port" -%% Latency stats: can get them on default PG port 5432 when not provided -2.4|t|t -D1| -D2| -D3| diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql deleted file mode 100644 index 8114673..0000000 --- a/test/CDB_FederatedServerListRemote.sql +++ /dev/null @@ -1,319 +0,0 @@ --- =================================================================== --- create FDW objects --- =================================================================== -\set QUIET on -SET client_min_messages TO error; -\set VERBOSITY terse -CREATE EXTENSION postgres_fdw; - -CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; -CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; - --- Create database to be used as remote -CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; - -SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - -SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback2'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - --- =================================================================== --- Setup 1 --- =================================================================== -\c cdb_fs_tester cdb_fs_tester - -CREATE TYPE user_enum AS ENUM ('foo', 'bar', 'buz'); -CREATE SCHEMA "S 1"; -CREATE TABLE "S 1"."T 1" ( - "C 1" int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 timestamptz, - c5 timestamp, - c6 varchar(10), - c7 char(10), - c8 user_enum, - CONSTRAINT t1_pkey PRIMARY KEY ("C 1") -); -CREATE TABLE "S 1"."T 2" ( - c1 int NOT NULL, - c2 text, - CONSTRAINT t2_pkey PRIMARY KEY (c1) -); -CREATE TABLE "S 1"."T 3" ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - CONSTRAINT t3_pkey PRIMARY KEY (c1) -); -CREATE TABLE "S 1"."T 4" ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - CONSTRAINT t4_pkey PRIMARY KEY (c1) -); - --- Disable autovacuum for these tables to avoid unexpected effects of that -ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false'); -ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false'); -ALTER TABLE "S 1"."T 3" SET (autovacuum_enabled = 'false'); -ALTER TABLE "S 1"."T 4" SET (autovacuum_enabled = 'false'); - -\c contrib_regression postgres -SET client_min_messages TO notice; -\set VERBOSITY terse -\set QUIET off - - --- =================================================================== --- Test listing remote schemas --- =================================================================== -\echo '## Test listing of remote schemas without permissions before the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); -\c contrib_regression postgres - -\echo '## Test listing of remote schemas (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); - -\echo '## Test listing of remote schemas without permissions after the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); -\c contrib_regression postgres - -\echo '## Test listing of remote schemas with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); -\c contrib_regression postgres - -\echo '## Test listing of remote schemas without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); -\c contrib_regression postgres - -\echo '## Test listing of remote schemas (rainy day): Server does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist'); - - --- =================================================================== --- Test listing remote tables --- =================================================================== - -\echo '## Test listing of remote tables without permissions before the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote tables (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); - -\echo '## Test listing of remote tables without permissions after the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote tables with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote tables without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote tables (rainy day): Server does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1'); - -\echo '## Test listing of remote tables (rainy day): Remote schema does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist'); - - --- =================================================================== --- Test listing remote columns --- =================================================================== - -\echo '## Test listing of remote columns without permissions before the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote columns (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); - -\echo '## Test listing of remote columns without permissions after the first instantiation (rainy day)' -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote columns with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote columns without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -\c contrib_regression postgres - -\echo '## Test listing of remote columns (rainy day): Server does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1'); - -\echo '## Test listing of remote columns (rainy day): Remote schema does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1'); - -\echo '## Test listing of remote columns (rainy day): Remote table does not exist' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist'); - -\echo '## Test listing of remote columns (rainy day): Remote table is NULL' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => NULL::text); - - --- =================================================================== --- Test that using a different user to list tables and dropping it --- does not break the server: We use loopback2 as it's in a clean state --- =================================================================== - - -\echo '## Test listing of remote objects with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester2'::name); -\c contrib_regression cdb_fs_tester2 -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); - -\c contrib_regression postgres -\echo '## Test that dropping the granted user works fine (sunny day)' -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; -DROP ROLE cdb_fs_tester2; - -\echo '## Test listing of remote objects with other user still works (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); - - --- =================================================================== --- Cleanup 1 --- =================================================================== -\set QUIET on - -\c cdb_fs_tester cdb_fs_tester -DROP TABLE "S 1". "T 1"; -DROP TABLE "S 1". "T 2"; -DROP TABLE "S 1". "T 3"; -DROP TABLE "S 1". "T 4"; - -DROP SCHEMA "S 1"; -DROP TYPE user_enum; - - --- =================================================================== --- Setup 2: Using Postgis too --- =================================================================== - -\c cdb_fs_tester postgres - -CREATE EXTENSION postgis; - -\c cdb_fs_tester cdb_fs_tester - -CREATE SCHEMA "S 1"; -CREATE TABLE "S 1"."T 5" ( - geom geometry(Geometry,4326), - geom_wm geometry(GeometryZ,3857), - geo_nosrid geometry, - geog geography -); - -\c contrib_regression postgres -SET client_min_messages TO notice; -\set VERBOSITY terse -\set QUIET off - - --- =================================================================== --- Test the listing functions --- =================================================================== - -\echo '## Test listing of remote geometry columns (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); -\echo '## Test listing of remote geometry columns (sunny day) - Rerun' --- Rerun should be ok -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); - - --- =================================================================== --- Test invalid password --- =================================================================== - -\echo '## Check error message with invalid password (rainy day)' -SELECT cartodb.CDB_Federated_Server_Register_PG(server => 'loopback_invalid'::text, config => '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "wrong password" - } -}'::jsonb); - -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid'); - -SELECT cartodb.CDB_Federated_Server_Unregister(server => 'loopback_invalid'::text); - --- =================================================================== --- Cleanup 2 --- =================================================================== -\set QUIET on - -\c cdb_fs_tester cdb_fs_tester -DROP TABLE "S 1". "T 5"; - -DROP SCHEMA "S 1"; - -\c contrib_regression postgres -\set QUIET on -SET client_min_messages TO error; -\set VERBOSITY terse - -SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); -SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'loopback2'::text); - -DROP DATABASE cdb_fs_tester; - --- Drop role -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; -DROP ROLE cdb_fs_tester; - -DROP EXTENSION postgres_fdw; - -\set QUIET off diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect deleted file mode 100644 index 244203c..0000000 --- a/test/CDB_FederatedServerListRemote_expect +++ /dev/null @@ -1,162 +0,0 @@ -C1| -C2| -## Test listing of remote schemas without permissions before the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote schemas (sunny day) -S 1 -information_schema -public -## Test listing of remote schemas without permissions after the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote schemas with permissions (sunny day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -S 1 -information_schema -public -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote schemas without permissions after revoking access (rainy day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote schemas (rainy day): Server does not exist -ERROR: Server "Does Not Exist" does not exist -## Test listing of remote tables without permissions before the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote tables (sunny day) -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" -f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] -f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] -f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -## Test listing of remote tables without permissions after the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote tables with permissions (sunny day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" -f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] -f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] -f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote tables without permissions after revoking access (rainy day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote tables (rainy day): Server does not exist -ERROR: Server "Does Not Exist" does not exist -## Test listing of remote tables (rainy day): Remote schema does not exist -## Test listing of remote columns without permissions before the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote columns (sunny day) -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" -C 1|integer -c2|integer -c3|text -c4|timestamp with time zone -c5|timestamp without time zone -c6|character varying -c7|character -c8|USER-DEFINED -## Test listing of remote columns without permissions after the first instantiation (rainy day) -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote columns with permissions (sunny day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" -C 1|integer -c2|integer -c3|text -c4|timestamp with time zone -c5|timestamp without time zone -c6|character varying -c7|character -c8|USER-DEFINED -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote columns without permissions after revoking access (rainy day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -You are now connected to database "contrib_regression" as user "postgres". -## Test listing of remote columns (rainy day): Server does not exist -ERROR: Server "Does Not Exist" does not exist -## Test listing of remote columns (rainy day): Remote schema does not exist -## Test listing of remote columns (rainy day): Remote table does not exist -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" -## Test listing of remote columns (rainy day): Remote table is NULL -ERROR: Remote table name cannot be NULL -## Test listing of remote objects with permissions (sunny day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -S 1 -information_schema -public -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" -f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] -f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] -f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" -C 1|integer -c2|integer -c3|text -c4|timestamp with time zone -c5|timestamp without time zone -c6|character varying -c7|character -c8|USER-DEFINED -You are now connected to database "contrib_regression" as user "postgres". -## Test that dropping the granted user works fine (sunny day) -REVOKE -DROP ROLE -## Test listing of remote objects with other user still works (sunny day) - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -S 1 -information_schema -public -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" -f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] -f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] -f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] -INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" -C 1|integer -c2|integer -c3|text -c4|timestamp with time zone -c5|timestamp without time zone -c6|character varying -c7|character -c8|USER-DEFINED -## Test listing of remote geometry columns (sunny day) -geo_nosrid|GEOMETRY,0 -geog|Geometry,0 -geom|GEOMETRY,4326 -geom_wm|GEOMETRY,3857 -## Test listing of remote geometry columns (sunny day) - Rerun -geo_nosrid|GEOMETRY,0 -geog|Geometry,0 -geom|GEOMETRY,4326 -geom_wm|GEOMETRY,3857 -## Check error message with invalid password (rainy day) - -ERROR: could not connect to server "cdb_fs_loopback_invalid" - -D1| -D2| diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql deleted file mode 100644 index 0982398..0000000 --- a/test/CDB_FederatedServerTables.sql +++ /dev/null @@ -1,405 +0,0 @@ --- =================================================================== --- create FDW objects --- =================================================================== -\set QUIET on -SET client_min_messages TO error; -\set VERBOSITY terse - -CREATE EXTENSION postgres_fdw; - --- We create a username following the same steps as organization members -CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; - -CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; -GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; - --- Create database to be used as remote -CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; - -SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ - "server": { - "host": "localhost", - "port": @@PGPORT@@ - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); - - --- =================================================================== --- create objects used through FDW loopback server --- =================================================================== - -\c cdb_fs_tester postgres - -CREATE EXTENSION postgis; - -\c cdb_fs_tester cdb_fs_tester - -CREATE SCHEMA remote_schema; -CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326)); - -INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry); -INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry); - -CREATE TABLE remote_schema.remote_geom2(cartodb_id bigint, another_field text, the_geom geometry(Geometry,4326), the_geom_webmercator geometry(Geometry,3857)); - -INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)'); - -CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text); -INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes'); - -CREATE TABLE remote_schema.remote_geom3(id bigint, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); - --- =================================================================== --- Test the listing functions --- =================================================================== - -\c contrib_regression postgres -SET client_min_messages TO error; -\set VERBOSITY terse -\set QUIET off - -\echo '## Registering an existing table works' -SELECT 'R1', cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom' - ); - -SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback.remote_geom; - -Select * FROM CDB_Federated_Server_List_Remote_Tables( - server => 'loopback', - remote_schema => 'remote_schema' -); - -\echo '## Registering another existing table works' -SELECT 'R2', cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom2', - id_column => 'cartodb_id', - geom_column => 'the_geom', - webmercator_column => 'the_geom_webmercator', - local_name => 'myFullTable' - ); - -SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback."myFullTable"; -Select * FROM CDB_Federated_Server_List_Remote_Tables( - server => 'loopback', - remote_schema => 'remote_schema' -); - -\echo '## Re-registering a table works' -SELECT 'R3', cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom2', - id_column => 'cartodb_id', - -- Switch geom and geom_column on purpose to force ST_Transform to be used - geom_column => 'the_geom_webmercator', - webmercator_column => 'the_geom', - local_name => 'different_name' - ); - --- The old view should dissapear -SELECT 'S3_old', cartodb_id, another_field FROM cdb_fs_loopback."myFullTable"; --- And the new appear -SELECT 'S3_new', cartodb_id, another_field FROM cdb_fs_loopback.different_name; - -\echo '## Unregistering works' --- Deregistering the first table -SELECT 'U1', cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom' - ); --- Selecting from the created view should fail now -SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; - -Select * FROM CDB_Federated_Server_List_Remote_Tables( - server => 'loopback', - remote_schema => 'remote_schema' -); - --- =================================================================== --- Test input --- =================================================================== - -\echo '## Registering a table: Invalid server fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'Does not exist', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: NULL server fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => NULL::text, - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: Invalid schema fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'Does not exist', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: NULL schema fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => NULL::text, - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: Invalid table fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'Does not exist', - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: NULL table fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => NULL::text, - id_column => 'id', - geom_column => 'geom' - ); - -\echo '## Registering a table: Invalid id fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'Does not exist', - geom_column => 'geom' - ); - -\echo '## Registering a table: NULL id fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => NULL::text, - geom_column => 'geom' - ); - -\echo '## Registering a table: Invalid geom_column fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'Does not exists' - ); - -\echo '## Registering a table: NULL geom_column is OK: Reuses geom_mercator' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => NULL::text, - webmercator_column => 'geom' - ); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; - -SELECT cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom' - ); - -\echo '## Registering a table: Invalid webmercator_column fails' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - webmercator_column => 'Does not exists' - ); - -\echo '## Registering a table: NULL webmercator_column is OK: Reuses geom' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - webmercator_column => NULL::text - ); -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; -SELECT cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom' - ); - -\echo '##Registering a table with extra columns show the correct information' -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom3', - id_column => 'id', - geom_column => 'geom', - webmercator_column => 'geom_mercator' - ); -Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom3'; -SELECT cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom3' - ); - --- =================================================================== --- Test conflicts --- =================================================================== - -\echo '## Target conflict is handled nicely: Table' -CREATE TABLE cdb_fs_loopback.localtable (a integer); -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - local_name => 'localtable'); - -\echo '## Target conflict is handled nicely: View' -CREATE VIEW cdb_fs_loopback.localtable2 AS Select * from cdb_fs_loopback.localtable; -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - local_name => 'localtable2'); - -DROP VIEW cdb_fs_loopback.localtable2; -DROP TABLE cdb_fs_loopback.localtable; - --- =================================================================== --- Test permissions --- =================================================================== - - -\echo '## Registering tables does not work without permissions' -\c contrib_regression cdb_fs_tester -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - local_name => 'localtable'); - -\echo '## Normal users can not write in the import schema' -CREATE TABLE cdb_fs_loopback.localtable (a integer); - -\echo '## Listing remote tables does not work without permissions' -Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); - -\echo '## Registering tables works with granted permissions' -\c contrib_regression postgres -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); -\c contrib_regression cdb_fs_tester -SELECT cartodb.CDB_Federated_Table_Register( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom', - id_column => 'id', - geom_column => 'geom', - local_name => 'localtable'); - -\echo '## Listing remote tables works with granted permissions' -Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); - -\echo '## Selecting from a registered table with granted permissions works' -Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; - -\echo '## Selecting from a registered table without permissions does not work' -\c contrib_regression cdb_fs_tester2 -CREATE OR REPLACE FUNCTION catch_permission_error(query text) -RETURNS bool -AS $$ -BEGIN - EXECUTE query; - RETURN FALSE; -EXCEPTION - WHEN insufficient_privilege THEN - RETURN TRUE; - WHEN OTHERS THEN - RAISE WARNING 'Exception %', sqlstate; - RETURN FALSE; -END -$$ LANGUAGE 'plpgsql'; -Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable$$); -DROP FUNCTION catch_permission_error(text); - -\echo '## Deleting a registered table without permissions does not work' -SELECT cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom' - ); - -\echo '## Only the owner can grant permissions over the server' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); - -\echo '## Everything works for a different user when granted permissions' -\c contrib_regression postgres -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); -\c contrib_regression cdb_fs_tester2 -Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); -Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; - -\echo '## A different user can unregister a table' -SELECT cartodb.CDB_Federated_Table_Unregister( - server => 'loopback', - remote_schema => 'remote_schema', - remote_table => 'remote_geom' - ); -Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); - -\echo '## Only the owner can revoke permissions over the server' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); - --- =================================================================== --- Cleanup --- =================================================================== - -\set QUIET on -\c contrib_regression postgres -SET client_min_messages TO error; -\set VERBOSITY terse - -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; -DROP ROLE cdb_fs_tester2; - -SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); -DROP DATABASE cdb_fs_tester; -REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; -DROP ROLE cdb_fs_tester; -DROP EXTENSION postgres_fdw; -\set QUIET off diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect deleted file mode 100644 index 6e02023..0000000 --- a/test/CDB_FederatedServerTables_expect +++ /dev/null @@ -1,116 +0,0 @@ -C1| -## Registering an existing table works -R1| -S1|1|POINT(1 1)|patata -S1|2|POINT(2 2)|patata2 -t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -## Registering another existing table works -R2| -S2|3|POINT(3 3)|patata -t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback."myFullTable"|cartodb_id|the_geom|the_geom_webmercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -## Re-registering a table works -R3| -ERROR: relation "cdb_fs_loopback.myFullTable" does not exist at character 49 -S3_new|3|patata -## Unregistering works -U1| -ERROR: relation "remote_geom" does not exist at character 71 -f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -## Registering a table: Invalid server fails -ERROR: Server "Does not exist" does not exist -## Registering a table: NULL server fails -ERROR: Server name cannot be NULL -## Registering a table: Invalid schema fails -ERROR: Could not import schema "Does not exist" of server "loopback": schema "Does not exist" is not present on foreign server "cdb_fs_loopback" -## Registering a table: NULL schema fails -ERROR: Schema name cannot be NULL -## Registering a table: Invalid table fails -ERROR: Could not import table "remote_schema.Does not exist" of server "loopback" -## Registering a table: NULL table fails -ERROR: Remote table name cannot be NULL -## Registering a table: Invalid id fails -ERROR: non integer id_column "Does not exist" -## Registering a table: NULL id fails -ERROR: non integer id_column "" -## Registering a table: Invalid geom_column fails -ERROR: non geometry column "Does not exists" -## Registering a table: NULL geom_column is OK: Reuses geom_mercator - -t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] - -## Registering a table: Invalid webmercator_column fails -ERROR: non geometry column "Does not exists" -## Registering a table: NULL webmercator_column is OK: Reuses geom - -t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] - -##Registering a table with extra columns show the correct information - -t|remote_geom3|cdb_fs_loopback.remote_geom3|id|geom|geom_mercator|[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] - -## Target conflict is handled nicely: Table -CREATE TABLE -ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists -## Target conflict is handled nicely: View -CREATE VIEW -ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable2" already exists -DROP VIEW -DROP TABLE -## Registering tables does not work without permissions -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to access the server "loopback" -## Normal users can not write in the import schema -ERROR: permission denied for schema cdb_fs_loopback at character 14 -## Listing remote tables does not work without permissions -ERROR: Not enough permissions to access the server "loopback" -## Registering tables works with granted permissions -You are now connected to database "contrib_regression" as user "postgres". - -You are now connected to database "contrib_regression" as user "cdb_fs_tester". - -## Listing remote tables works with granted permissions -t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -## Selecting from a registered table with granted permissions works -1|POINT(1 1) -2|POINT(2 2) -## Selecting from a registered table without permissions does not work -You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -CREATE FUNCTION -t -DROP FUNCTION -## Deleting a registered table without permissions does not work -ERROR: Not enough permissions to access the server "loopback" -## Only the owner can grant permissions over the server -ERROR: You do not have rights to grant access on "loopback" -## Everything works for a different user when granted permissions -You are now connected to database "contrib_regression" as user "postgres". - -You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -1|POINT(1 1) -2|POINT(2 2) -## A different user can unregister a table -NOTICE: drop cascades to view cdb_fs_loopback.localtable - -f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] -f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] -f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] -## Only the owner can revoke permissions over the server -ERROR: You do not have rights to revoke access on "loopback" -D1| diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect deleted file mode 100644 index 836646d..0000000 --- a/test/CDB_FederatedServer_expect +++ /dev/null @@ -1,69 +0,0 @@ -## List empty servers shows nothing -## List non-existent server shows nothing -## Create and list a server works -1.3| -1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -## Create and list a second server works -2.1| -2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) -## List server by name works -2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -## Re-register a second server works -3.1| -3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) -## Unregister server 1 works -4.1| -4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) -## Unregistering a server that does not exist fails -ERROR: Server "doesNotExist" does not exist -## Unregister the second server works -6.1| -## Create a server with NULL name fails -ERROR: Server name cannot be NULL -## Create a server with NULL config fails -7.01| -## Create a server with empty config fails -ERROR: Server information is mandatory -## Create a server without credentials fails -ERROR: Credentials are mandatory -## Create a server with empty credentials works -7.3| -7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,) -7.5| -## Create a server without options fails -ERROR: Server information is mandatory -## Create a server with special characters works -8.1| -8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") -8.3| -9.1| -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -## All users are able to list servers -9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,) -## Only superadmins can create servers -ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw -You are now connected to database "contrib_regression" as user "postgres". -## Granting access to a user works -9.5| -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -9.55|(myRemote3,postgres_fdw,localhost,5432,,read-only,fdw_user) -You are now connected to database "contrib_regression" as user "postgres". -ERROR: Server "does not exist" does not exist -ERROR: Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist -## Granting access again raises a notice -NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" -9.8| -## Revoking access to a user works -9.9| -9.10| -## Unregistering a server with active grants works -9.11| -## A user with granted access can not drop a server -10.1| -10.2| -You are now connected to database "contrib_regression" as user "cdb_fs_tester". -ERROR: Not enough permissions to drop the server "myRemote4" -You are now connected to database "contrib_regression" as user "postgres". -10.4| diff --git a/test/CDB_SyncTableTest.sql b/test/CDB_SyncTableTest.sql index 5b945ae..fc2711e 100644 --- a/test/CDB_SyncTableTest.sql +++ b/test/CDB_SyncTableTest.sql @@ -51,7 +51,7 @@ UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecodin SET client_min_messages TO notice; SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); SELECT * FROM test_sync_source ORDER BY cartodb_id; -SELECT cartodb_id, the_geom, lat, lon, name FROM test_sync_dest ORDER BY cartodb_id; +SELECT * FROM test_sync_dest ORDER BY cartodb_id; \echo 'It will work with schemas that need quoting' \set QUIET on diff --git a/test/CDB_SyncTableTest_expect b/test/CDB_SyncTableTest_expect index 9e0c8df..01cb358 100644 --- a/test/CDB_SyncTableTest_expect +++ b/test/CDB_SyncTableTest_expect @@ -54,10 +54,10 @@ NOTICE: MODIFIED 0 row(s) 2|||2|2|bar 4|||4|4|cantaloupe 5|||5|5|sandia -1|0101000020E6100000000000000000F03F000000000000F03F|1|1|foo -2|0101000020E610000000000000000000400000000000000040|2|2|bar -4|0101000020E610000000000000000010400000000000001040|4|4|cantaloupe -5|0101000020E610000000000000000014400000000000001440|5|5|sandia +1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo +2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar +4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe +5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia It will work with schemas that need quoting INSERT 0 1 diff --git a/test/CDB_Username_expect b/test/CDB_Username_expect index d030a42..fe1ea5b 100644 --- a/test/CDB_Username_expect +++ b/test/CDB_Username_expect @@ -1,4 +1,4 @@ -@@PGUSER@@ +postgres fulano fulanito diff --git a/test/extension/test.sh b/test/extension/test.sh index aab19f3..faf3d93 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -598,6 +598,68 @@ test_extension|public|"local-table-with-dashes"' sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;" sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't' + + # Check user-defined FDW's + # Set up a user foreign server + read -d '' ufdw_config <<- EOF +{ + "server": { + "extensions": "postgis", + "dbname": "fdw_target", + "host": "localhost", + "port": ${PGPORT:-5432} + }, + "user_mapping": { + "user": "fdw_user", + "password": "foobarino" + } +} +EOF + sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');" + + # Grant a user access to that FDW, and to grant to others + sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;' + + # Set up a user foreign table + sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');" + + # Check that the table can be accessed by the owner/creator + sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;' + sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 + + # Check that a role with no permissions cannot use the FDW to access a remote table + sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails + + # Check that the table can be accessed by some other user by granting the role + sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails + sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;' + sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 + sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;' + + # Check that the table can be accessed by org members + sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails + sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');" + sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 + sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');" + + # By default publicuser cannot access the FDW + sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails + sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted + sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 + sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;' + + # If there are dependent objects, we cannot drop the foreign server + sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails + sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;' + sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" + + # But if there are, we can set the force flag to true to drop everything (defaults to false) + sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');" + sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;' + sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');" + sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)" + + # Teardown DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;' From 654fa3ac02b71798d4c2cfac3cf50492c8ce37d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 23 Dec 2019 20:25:36 +0100 Subject: [PATCH 2/6] Update NEWS.md Co-Authored-By: Daniel G. Aubert --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index a1f5890..2413dd3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,5 @@ 0.34.0 (2019-12-23) -* Revert changes done in 0.32.0, keeping function signature to drop them +* Revert changes done in 0.33.0, keeping function signature to drop them 0.33.0 (2019-12-20) * Revert `Make PG12 depend on plpython3u instead of plpythonu`. From 8974986d61e702f882843c77cd14d088a709a281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 23 Dec 2019 20:34:11 +0100 Subject: [PATCH 3/6] Release 0.34.0 --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12ebc9c..439e93e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.33.0 +EXTVERSION = 0.34.0 SED = sed AWK = awk @@ -106,6 +106,7 @@ UPGRADABLE = \ 0.31.0 \ 0.32.0 \ 0.33.0 \ + 0.34.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) From 901064232652a905945eb6154f29490cb545ea68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 24 Dec 2019 00:33:35 +0100 Subject: [PATCH 4/6] Revert "Revert 0.33.0" This reverts commit 24cb6cf9c150150e256a92ea9f59a7d9a2407195. --- NEWS.md | 3 + scripts-available/CDB_FederatedServer.sql | 452 +++++++++++++++++- .../CDB_FederatedServerDiagnostics.sql | 252 +++++++++- .../CDB_FederatedServerListRemote.sql | 305 +++++++++++- .../CDB_FederatedServerTables.sql | 354 +++++++++++++- scripts-available/CDB_ForeignTable.sql | 179 +------ scripts-available/CDB_Groups_API.sql | 1 - scripts-available/CDB_Organizations.sql | 26 +- sql/test_setup.sql | 2 +- test/CDB_CartodbfyTableTest.sql | 4 +- test/CDB_CartodbfyTableTest_expect | 4 +- test/CDB_FederatedServer.sql | 220 +++++++++ test/CDB_FederatedServerDiagnostics.sql | 125 +++++ test/CDB_FederatedServerDiagnostics_expect | 28 ++ test/CDB_FederatedServerListRemote.sql | 319 ++++++++++++ test/CDB_FederatedServerListRemote_expect | 162 +++++++ test/CDB_FederatedServerTables.sql | 405 ++++++++++++++++ test/CDB_FederatedServerTables_expect | 116 +++++ test/CDB_FederatedServer_expect | 69 +++ test/CDB_SyncTableTest.sql | 2 +- test/CDB_SyncTableTest_expect | 8 +- test/CDB_Username_expect | 2 +- test/extension/test.sh | 62 --- 23 files changed, 2793 insertions(+), 307 deletions(-) create mode 100644 test/CDB_FederatedServer.sql create mode 100644 test/CDB_FederatedServerDiagnostics.sql create mode 100644 test/CDB_FederatedServerDiagnostics_expect create mode 100644 test/CDB_FederatedServerListRemote.sql create mode 100644 test/CDB_FederatedServerListRemote_expect create mode 100644 test/CDB_FederatedServerTables.sql create mode 100644 test/CDB_FederatedServerTables_expect create mode 100644 test/CDB_FederatedServer_expect diff --git a/NEWS.md b/NEWS.md index 2413dd3..ab968d0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +0.35.0 (XXXX-XX-XX) +* Reapply the changes in 0.33.0 (the issue we were looking for was unrelated) + 0.34.0 (2019-12-23) * Revert changes done in 0.33.0, keeping function signature to drop them diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index a7885a5..618bedd 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -1,15 +1,437 @@ -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Name_Pattern(); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Name(TEXT, BOOL); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Extract_Server_Name(NAME); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Schema_Name(NAME, TEXT); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Role_Name(NAME); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Create_Schema(NAME, TEXT); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_server_type(NAME); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_credentials_to_user_mapping(JSONB); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_add_default_options(jsonb); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_get_usermapping_username(NAME); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Register_PG(TEXT, JSONB); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Unregister(TEXT); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Servers(TEXT); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Grant_Access(TEXT, NAME); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Revoke_Access(TEXT, NAME); +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +-- +-- This function is just a placement to store and use the pattern for +-- foreign object names +-- Servers: cdb_fs_$(server_name) +-- View schema: cdb_fs_$(server_name) +-- > This is where all views created when importing tables are placed +-- > One server has only one view schema +-- Import Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) +-- > This is where the foreign tables are placed +-- > One server has one import schema per remote schema plus auxiliar ones used +-- to access the remote catalog (pg_catalog, information_schema...) +-- Owner role: cdb_fs_$(md5sum(current_database() || server_name) +-- > This is the role than owns all schemas and tables related to the server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() +RETURNS TEXT +AS $$ + SELECT 'cdb_fs_'::text; +$$ +LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Produce a valid DB name for servers generated for the Federated Server +-- If check_existence is true, it'll throw if the server doesn't exists +-- This name is also used as the schema to store views +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) +RETURNS NAME +AS $$ +DECLARE + internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); +BEGIN + IF input_name IS NULL OR char_length(input_name) = 0 THEN + RAISE EXCEPTION 'Server name cannot be NULL'; + END IF; + + -- We discard anything that would be truncated + IF (char_length(internal_server_name) >= 64) THEN + RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name; + END IF; + + IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN + RAISE EXCEPTION 'Server "%" does not exist', input_name; + END IF; + + RETURN internal_server_name::name; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- +-- Given the internal name for a remote server, it returns the name used by the user +-- Reverses __CDB_FS_Generate_Server_Name +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME) +RETURNS TEXT +AS $$ + SELECT right(internal_server_name, + char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; +$$ +LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Produce a valid name for a schema generated for the Federated Server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT) +RETURNS NAME +AS $$ +DECLARE + hash_value text := md5(internal_server_name::text || '__' || schema_name::text); +BEGIN + IF schema_name IS NULL THEN + RAISE EXCEPTION 'Schema name cannot be NULL'; + END IF; + RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- +-- Produce a valid name for a role generated for the Federated Server +-- This needs to include the current database in its hash to avoid collisions in clusters with more than one database +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME) +RETURNS NAME +AS $$ +DECLARE + hash_value text := md5(current_database()::text || '__' || internal_server_name::text); + role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value); +BEGIN + RETURN role_name::name; +END +$$ +LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +-- +-- Creates (if not exist) a schema to place the objects for a remote schema +-- The schema is with the same AUTHORIZATION as the server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT) +RETURNS NAME +AS $$ +DECLARE + schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); +BEGIN + -- By changing the local role to the owner of the server we have an + -- easy way to check for permissions and keep all objects under the same owner + BEGIN + EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); + EXCEPTION + WHEN invalid_parameter_value THEN + RAISE EXCEPTION 'Server "%" does not exist', + @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); + WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); + END; + + IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN + EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); + END IF; + RETURN schema_name; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Returns the type of a server by internal name +-- Currently all of them should be postgres_fdw +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME) +RETURNS name +AS $$ + SELECT f.fdwname + FROM pg_foreign_server s + JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid + WHERE s.srvname = internal_server_name; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + +-- +-- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB) +RETURNS jsonb +AS $$ +DECLARE + mapping jsonb := '{}'::jsonb; +BEGIN + IF NOT (input_config ? 'credentials') THEN + RAISE EXCEPTION 'Credentials are mandatory'; + END IF; + + -- For now, allow not passing username or password + IF input_config->'credentials'->'username' IS NOT NULL THEN + mapping := jsonb_build_object('user', input_config->'credentials'->'username'); + END IF; + IF input_config->'credentials'->'password' IS NOT NULL THEN + mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password'); + END IF; + + RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping); +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Take a config jsonb as input and return it augmented with default +-- options +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb) +RETURNS jsonb +AS $$ +DECLARE + default_options jsonb := '{ + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }'; + server_config jsonb; +BEGIN + IF NOT (input_config ? 'server') THEN + RAISE EXCEPTION 'Server information is mandatory'; + END IF; + server_config := default_options || to_jsonb(input_config->'server'); + RETURN jsonb_set(input_config, '{server}'::text[], server_config); +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Given an server name, returns the username used in the configuration if the caller has rights to access it +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME) +RETURNS text +AS $$ +DECLARE + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); + username text; +BEGIN + BEGIN + EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); + EXCEPTION WHEN OTHERS THEN + RETURN NULL; + END; + + SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username + FROM pg_foreign_server s + LEFT JOIN pg_user_mappings u + ON u.srvid = s.oid + WHERE s.srvname = internal_server_name + ORDER BY 1; + + RESET ROLE; + + RETURN username; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + + +-- +-- Registers a new PG server +-- +-- Example config: '{ +-- "server": { +-- "dbname": "fdw_target", +-- "host": "localhost", +-- "port": 5432, +-- "extensions": "postgis", +-- "updatable": "false", +-- "use_remote_estimate": "true", +-- "fetch_size": "1000" +-- }, +-- "credentials": { +-- "username": "fdw_user", +-- "password": "foobarino" +-- } +-- }' +-- +-- The configuration from __CDB_FS_add_default_options will be appended +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); + final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + row record; + option record; +BEGIN + IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN + RAISE EXCEPTION 'postgres_fdw extension is not installed' + USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`'; + END IF; + + -- We only create server and roles if the server didn't exist before + IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN + BEGIN + EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN + EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); + END IF; + EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); + + -- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases + -- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database + EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name); + EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name); + EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name); + EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name); + EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name); + + EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); + EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); + EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM + USING HINT = 'Please clean the left over objects'; + END; + END IF; + + -- Add new options + FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p + LOOP + IF NOT EXISTS ( + WITH a AS ( + SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal + ) SELECT * from a where options = row.key) + THEN + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value); + ELSE + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value); + END IF; + END LOOP; + + -- Update user mapping settings + FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o + LOOP + IF NOT EXISTS ( + WITH a AS ( + SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name + ) SELECT * from a where options = option.key) + THEN + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value); + ELSE + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value); + END IF; + END LOOP; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Drops a registered server and all the objects associated with it +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); +BEGIN + SET client_min_messages = ERROR; + BEGIN + EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); + EXECUTE FORMAT ('DROP OWNED BY %I', role_name); + EXECUTE FORMAT ('DROP ROLE %I', role_name); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List registered servers +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') +RETURNS TABLE ( + name text, + driver text, + host text, + port text, + dbname text, + readmode text, + username text +) +AS $$ +DECLARE + server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server); +BEGIN + RETURN QUERY SELECT + -- Name as shown to the user + @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", + + -- Which driver are we using (postgres_fdw, odbc_fdw...) + @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver", + + -- Read options from pg_foreign_server + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", + CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", + + @extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username" + FROM pg_foreign_server s + LEFT JOIN pg_user_mappings u + ON u.srvid = s.oid + WHERE s.srvname ILIKE server_name + ORDER BY 1; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE; + + +-- +-- Grant access to a server +-- In the future we might consider adding the server's view schema to the role search_path +-- to make it easier to access the created views +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); +BEGIN + IF (db_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" cannot be NULL', username; + END IF; + BEGIN + EXECUTE format('GRANT %I TO %I', server_role_name, db_role); + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION 'You do not have rights to grant access on "%"', server; + WHEN OTHERS THEN + RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM; + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Revoke access to a server +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); +BEGIN + IF (db_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" cannot be NULL', username; + END IF; + BEGIN + EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role); + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server; + WHEN OTHERS THEN + RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM; + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index b05b6d3..b1e2e4c 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -1,9 +1,243 @@ -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Import_If_Not_Exists(name, name, name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Version_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Options_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Host_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Port_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(name, float, integer); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Server_Diagnostics_PG(name); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Diagnostics(TEXT); +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +-- +-- Import a foreign table if it does not exist +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name) +RETURNS void +AS $$ +DECLARE + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + remote_schema, remote_table, server_internal, local_schema); + END IF; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Get the version of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name) +RETURNS text +AS $$ +DECLARE + remote_schema name := 'pg_catalog'; + remote_table name := 'pg_settings'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + remote_server_version text; +BEGIN + PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); + + BEGIN + EXECUTE format(' + SELECT setting FROM %I.%I WHERE name = ''server_version''; + ', local_schema, remote_table) INTO remote_server_version; + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + END; + + RETURN remote_server_version; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-- +-- Get the PostGIS extension version of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name) +RETURNS text +AS $$ +DECLARE + remote_schema name := 'pg_catalog'; + remote_table name := 'pg_extension'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + remote_postgis_version text; +BEGIN + PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); + + BEGIN + EXECUTE format(' + SELECT extversion FROM %I.%I WHERE extname = ''postgis''; + ', local_schema, remote_table) INTO remote_postgis_version; + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + END; + + RETURN remote_postgis_version; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-- +-- Get the foreign server options of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name) +RETURNS jsonb +AS $$ + -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html + -- See https://www.postgresql.org/docs/current/functions-info.html + SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server + WHERE srvname = server_internal + ) AS opt; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + + +-- +-- Get a foreign PG server hostname from the catalog +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name) +RETURNS text +AS $$ + SELECT option_value FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server WHERE srvname = server_internal + ) AS opt WHERE opt.option_name = 'host'; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + + +-- +-- Get a foreign PG server port from the catalog +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name) +RETURNS integer +AS $$ + SELECT option_value::integer FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server WHERE srvname = server_internal + ) AS opt WHERE opt.option_name = 'port'; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + +-- +-- Get one measure of network latency in ms to a remote TCP server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency( + server_internal name, + timeout_seconds float DEFAULT 5.0, + n_samples integer DEFAULT 10 +) +RETURNS jsonb +AS $$ + import socket + import json + import math + from timeit import default_timer as timer + + plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name']) + rv = plpy.execute(plan, [server_internal], 1) + host = rv[0]['host'] + + plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name']) + rv = plpy.execute(plan, [server_internal], 1) + port = rv[0]['port'] or 5432 + + n_errors = 0 + samples = [] + + for i in range(n_samples): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout_seconds) + + t_start = timer() + + try: + s.connect((host, int(port))) + t_stop = timer() + s.shutdown(socket.SHUT_RD) + except (socket.timeout, OSError, socket.error) as ex: + plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) + n_errors += 1 + finally: + s.close() + + t_connect = (t_stop - t_start) * 1000.0 + plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) + samples.append(t_connect) + + stats = { + 'n_samples': n_samples, + 'n_errors': n_errors, + } + n = len(samples) + if n >= 1: + mean = sum(samples) / n + stats.update({ + 'avg': round(mean, 3), + 'min': round(min(samples), 3), + 'max': round(max(samples), 3) + }) + if n >= 2: + var = sum([ (x - mean)**2 for x in samples ]) / (n-1) + stdev = math.sqrt(var) + stats.update({ + 'stdev': round(stdev, 3) + }) + return json.dumps(stats) +$$ +LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + + +-- +-- Collect and return diagnostics info from a remote PG into a jsonb +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name) +RETURNS jsonb +AS $$ +DECLARE + remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); + remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); + remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); + remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal); +BEGIN + RETURN jsonb_build_object( + 'server_version', remote_server_version, + 'postgis_version', remote_postgis_version, + 'server_options', remote_server_options, + 'server_latency_ms', remote_server_latency_ms + ); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + + +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + +-- +-- Collect and return diagnostics info from a remote PG into a jsonb +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT) +RETURNS jsonb +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 1632252..0367027 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -1,7 +1,298 @@ -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Schemas_PG(name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Tables_PG(name, name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Columns_PG(name, name); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(name, name, name); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Schemas(TEXT); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Tables(TEXT, TEXT); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Columns(TEXT, TEXT, TEXT); +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +-- +-- List the schemas of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name) +RETURNS TABLE(remote_schema name) +AS $$ +DECLARE + -- Import schemata from the information schema + -- + -- "The view schemata contains all schemas in the current database + -- that the current user has access to (by way of being the owner + -- or having some privilege)." + -- See https://www.postgresql.org/docs/11/infoschema-schemata.html + -- + -- "The information schema is defined in the SQL standard and can + -- therefore be expected to be portable and remain stable" + -- See https://www.postgresql.org/docs/11/information-schema.html + inf_schema name := 'information_schema'; + remote_table name := 'schemata'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); +BEGIN + -- Import the foreign schemata table + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_table, server_internal, local_schema); + END IF; + + -- Return the result we're interested in. Exclude toast and temp schemas + BEGIN + RETURN QUERY EXECUTE format(' + SELECT schema_name::name AS remote_schema FROM %I.%I + WHERE schema_name NOT LIKE %s + ORDER BY remote_schema + ', local_schema, remote_table, '''pg_%'''); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List the names of the tables in a remote PG schema +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name) +RETURNS TABLE(remote_table name) +AS $func$ +DECLARE + -- Import `tables` from the information schema + -- + -- "The view tables contains all tables and views defined in the + -- current database. Only those tables and views are shown that + -- the current user has access to (by way of being the owner or + -- having some privilege)." + -- https://www.postgresql.org/docs/11/infoschema-tables.html + + -- Create local target schema if it does not exists + inf_schema name := 'information_schema'; + remote_table name := 'tables'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); +BEGIN + -- Import the foreign `tables` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_table, server_internal, local_schema); + END IF; + + -- Note: in this context, schema names are not to be quoted + RETURN QUERY EXECUTE format($q$ + SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name + $q$, local_schema, remote_table, remote_schema); +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-- +-- List the columns in a remote PG schema +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name) +RETURNS TABLE(table_name name, column_name name, column_type text) +AS $func$ +DECLARE + -- Import `columns` from the information schema + -- + -- "The view columns contains information about all table columns (or view columns) + -- in the database. System columns (oid, etc.) are not included. Only those columns + -- are shown that the current user has access to (by way of being the owner or having some privilege)." + -- https://www.postgresql.org/docs/11/infoschema-columns.html + + -- Create local target schema if it does not exists + inf_schema name := 'information_schema'; + remote_col_table name := 'columns'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); +BEGIN + -- Import the foreign `columns` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_col_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_col_table, server_internal, local_schema); + END IF; + + -- Note: in this context, schema names are not to be quoted + -- We join with the geometry columns to change the type `USER-DEFINED` + -- by its appropiate geometry and srid + RETURN QUERY EXECUTE format($q$ + SELECT + a.table_name::name, + a.column_name::name, + COALESCE(b.column_type, a.data_type)::TEXT as column_type + FROM + %I.%I a + LEFT JOIN + @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b + ON a.table_name = b.table_name AND a.column_name = b.column_name + WHERE table_schema = '%s' + ORDER BY a.table_name, a.column_name $q$, + local_schema, remote_col_table, + server_internal, remote_schema, + remote_schema); +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List the geometry columns in a remote PG schema +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public') +RETURNS TABLE(table_name name, column_name name, column_type text) +AS $func$ +DECLARE + -- Import `geometry_columns` and `geography_columns` from the postgis schema + -- We assume that postgis is installed in the public schema + + -- Create local target schema if it does not exists + remote_geometry_view name := 'geometry_columns'; + remote_geography_view name := 'geography_columns'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema); +BEGIN + -- Import the foreign `geometry_columns` and `geography_columns` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_geometry_view + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I', + postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema); + END IF; + + BEGIN + -- Note: We return both the type and srid as the type + RETURN QUERY EXECUTE format($q$ + SELECT + f_table_name::NAME as table_name, + f_geometry_column::NAME as column_name, + type::TEXT || ',' || srid::TEXT as column_type + FROM + ( + SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I + ) _geo_views + WHERE f_table_schema = '%s' + $q$, + local_schema, remote_geometry_view, + local_schema, remote_geography_view, + remote_schema); + EXCEPTION WHEN OTHERS THEN + RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"', + postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + RETURN; + END; +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + +-- +-- List remote schemas in a federated server that the current user has access to. +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT) +RETURNS TABLE(remote_schema name) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List remote tables in a federated server that the current user has access to. +-- For registered tables it returns also the associated configuration +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT) +RETURNS TABLE( + registered boolean, + remote_table TEXT, + local_qualified_name TEXT, + id_column_name TEXT, + geom_column_name TEXT, + webmercator_column_name TEXT, + columns JSON + ) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY + SELECT + coalesce(registered_tables.registered, false)::boolean as registered, + foreign_tables.remote_table::text as remote_table, + registered_tables.local_qualified_name as local_qualified_name, + registered_tables.id_column_name as id_column_name, + registered_tables.geom_column_name as geom_column_name, + registered_tables.webmercator_column_name as webmercator_column_name, + remote_columns.columns as columns + FROM + @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables + LEFT JOIN + @extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables + ON foreign_tables.remote_table = registered_tables.remote_table + LEFT JOIN + ( -- Extract and group columns with their remote table + SELECT table_name, + json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns + FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) + GROUP BY table_name + ) remote_columns + ON foreign_tables.remote_table = remote_columns.table_name + ORDER BY foreign_tables.remote_table; + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List the columns of a remote table in a federated server that the current user has access to. +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( + server TEXT, + remote_schema TEXT, + remote_table TEXT) +RETURNS TABLE(column_n name, column_t text) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + IF remote_table IS NULL THEN + RAISE EXCEPTION 'Remote table name cannot be NULL'; + END IF; + + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY + SELECT + column_name, + column_type + FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) + WHERE table_name = remote_table + ORDER BY column_name; + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 5149644..0cc6385 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -1,9 +1,345 @@ -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Integer(REGCLASS, NAME); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Geometry(REGCLASS, NAME); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_GetColumns(REGCLASS); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_id_column(TEXT); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_geom_column(TEXT); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_webmercator_column(TEXT); -DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Registered_Tables(NAME,TEXT); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Register(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, NAME); -DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Unregister(TEXT, TEXT, TEXT); +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +-- +-- Checks if a column is of integer type +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid IN (SELECT oid FROM pg_type + WHERE typname IN + ('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8')); + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; + +-- +-- Checks if a column is of geometry type +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid = '@postgisschema@.geometry'::regtype; + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; + +-- +-- Returns the name of all the columns from a table +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS) +RETURNS SETOF NAME +AS $$ + SELECT + a.attname as "colname" + FROM pg_catalog.pg_attribute a + WHERE + a.attnum > 0 + AND NOT a.attisdropped + AND a.attrelid = ( + SELECT c.oid + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = input_table::oid + ) + ORDER BY a.attnum; +$$ LANGUAGE SQL STABLE PARALLEL UNSAFE; + +-- +-- Returns the id column from a view definition +-- Note: The id is always of one of this forms: +-- SELECT t.cartodb_id, +-- SELECT t.my_id as cartodb_id, +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def + ) + SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2)) + FROM column_definitions + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id') + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Returns the geom column from a view definition +-- +-- Note: The the_geom is always of one of this forms: +-- t.the_geom, +-- t.my_geom as the_geom, +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def + ) + SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) + FROM column_definitions + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom') + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Returns the webmercatorcolumn from a view definition +-- Note: The the_geom_webmercator is always of one of this forms: +-- t.the_geom_webmercator, +-- t.my_geom as the_geom_webmercator, +-- Or without the trailing comma: +-- t.the_geom_webmercator +-- t.my_geom as the_geom_webmercator +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def + ) + SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) + FROM column_definitions + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator') + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +-- +-- List all registered tables in a server + schema +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables( + server_internal NAME, + remote_schema TEXT + ) +RETURNS TABLE( + registered boolean, + remote_table TEXT, + local_qualified_name TEXT, + id_column_name TEXT, + geom_column_name TEXT, + webmercator_column_name TEXT + ) +AS $$ +DECLARE + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + RETURN QUERY SELECT + true as registered, + source_table::text as remote_table, + format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name, + @extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name, + @extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name, + @extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name + FROM + ( + SELECT DISTINCT + dependent_ns.nspname as dependent_schema, + dependent_view.relname as dependent_view, + source_table.relname as source_table, + pg_get_viewdef(dependent_view.oid) as view_definition + FROM pg_depend + JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid + JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid + JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid + JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace + JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace + WHERE + source_ns.nspname = local_schema + ORDER BY 1,2 + ) _aux; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + +-- +-- Sets up a Federated Table +-- +-- Precondition: the federated server has to be set up via +-- CDB_Federated_Server_Register_PG +-- +-- Postcondition: it generates a view in the schema of the user that +-- can be used through SQL and Maps API's. +-- If the table was already exported, it will be dropped and re-imported +-- +-- The view is placed under the server's view schema (cdb_fs_$(server_name)) +-- +-- E.g: +-- SELECT cartodb.CDB_SetUp_PG_Federated_Table( +-- 'amazon', -- mandatory, name of the federated server +-- 'my_remote_schema', -- mandatory, schema name +-- 'my_remote_table', -- mandatory, table name +-- 'id', -- mandatory, name of the id column +-- 'geom', -- optional, name of the geom column, preferably in 4326 +-- 'webmercator' -- optional, should be in 3857 if present +-- 'local_name' -- optional, name of the local view (uses the remote_name if not declared) +-- ); +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( + server TEXT, + remote_schema TEXT, + remote_table TEXT, + id_column TEXT, + geom_column TEXT DEFAULT NULL, + webmercator_column TEXT DEFAULT NULL, + local_name NAME DEFAULT NULL +) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + + src_table REGCLASS; -- import_schema.remote_table - import_schema is local_schema + local_view text; -- view_schema.local_name - view_schema is server_internal + + rest_of_cols TEXT[]; + geom_expression TEXT; + webmercator_expression TEXT; + carto_columns_expression TEXT[]; +BEGIN + IF remote_table IS NULL THEN + RAISE EXCEPTION 'Remote table name cannot be NULL'; + END IF; + + -- Make do with whatever columns are provided + IF webmercator_column IS NULL THEN + webmercator_column := geom_column; + ELSIF geom_column IS NULL THEN + geom_column := webmercator_column; + END IF; + + IF local_name IS NULL THEN + local_name := remote_table; + END IF; + + -- Import the foreign table + -- Drop the old view / table if there was one + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN + EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table); + END IF; + BEGIN + EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', + remote_schema, remote_table, server_internal, local_schema); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM; + END; + + BEGIN + src_table := format('%I.%I', local_schema, remote_table); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server; + END; + + -- Check id_column is numeric + IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN + RAISE EXCEPTION 'non integer id_column "%"', id_column; + END IF; + + -- Check if the geom and mercator columns have a geometry type (if provided) + IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN + RAISE EXCEPTION 'non geometry column "%"', geom_column; + END IF; + IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN + RAISE EXCEPTION 'non geometry column "%"', webmercator_column; + END IF; + + -- Get a list of columns excluding the id, geom and the_geom_webmercator + SELECT ARRAY( + SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c + WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL) + ) INTO rest_of_cols; + + IF geom_column IS NULL + THEN + geom_expression := 'NULL AS the_geom'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326 + THEN + geom_expression := format('t.%I AS the_geom', geom_column); + ELSE + -- It needs an ST_Transform to 4326 + geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column); + END IF; + + IF webmercator_column IS NULL + THEN + webmercator_expression := 'NULL AS the_geom_webmercator'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857 + THEN + webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column); + ELSE + -- It needs an ST_Transform to 3857 + webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column); + END IF; + + -- CARTO columns expressions + carto_columns_expression := ARRAY[ + format('t.%1$I AS cartodb_id', id_column), + geom_expression, + webmercator_expression + ]; + + -- Create view schema if it doesn't exist + IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN + EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name); + END IF; + + -- Create a view with homogeneous CDB fields + BEGIN + EXECUTE format( + 'CREATE OR REPLACE VIEW %1$I.%2$I AS + SELECT %3s + FROM %4$s t', + server_internal, local_name, + array_to_string(carto_columns_expression || rest_of_cols, ','), + src_table + ); + EXCEPTION WHEN OTHERS THEN + IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN + RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name; + ELSE + RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; + END IF; + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Unregisters a remote table. Any dependent object will be dropped +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister( + server TEXT, + remote_schema TEXT, + remote_table TEXT +) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index c86f5ef..b9449de 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -139,176 +139,6 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; --- Produce a valid DB name for objects created for the user FDW's -CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME) -RETURNS NAME AS $$ - -- Note on input we use %s and on output we use %I, in order to - -- avoid double escaping - SELECT format('cdb_fdw_%s', fdw_input_name)::NAME; -$$ -LANGUAGE sql IMMUTABLE PARALLEL SAFE; - --- A function to set up a user-defined foreign data server --- It does not read from CDB_Conf. --- Only superuser roles can invoke it successfully --- --- Sample call: --- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{ --- "server": { --- "extensions": "postgis", --- "dbname": "testdb", --- "host": "myhostname.us-east-2.rds.amazonaws.com", --- "port": "5432" --- }, --- "user_mapping": { --- "user": "fdw_user", --- "password": "secret" --- } --- }'); --- --- Underneath it will: --- * Set up postgresql_fdw --- * Create a server with the name 'cdb_fdw_amazon' --- * Create a role called 'cdb_fdw_amazon' to manage access --- * Create a user mapping with that role 'cdb_fdw_amazon' --- * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign --- tables over there --- --- It is the responsibility of the superuser to grant that role to either: --- * Nobody --- * Specific roles: GRANT amazon TO role_name; --- * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon'); --- * The publicuser: GRANT cdb_fdw_amazon TO publicuser; -CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json) -RETURNS void AS $$ -DECLARE - row record; - option record; - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); -BEGIN - -- TODO: refactor with original function - -- This function tries to be as idempotent as possible, by not creating anything more than once - -- (not even using IF NOT EXIST to avoid throwing warnings) - IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN - CREATE EXTENSION postgres_fdw; - RAISE NOTICE 'Created postgres_fdw EXTENSION'; - END IF; - -- Create FDW first if it does not exist - IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name) - THEN - EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name); - RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name; - END IF; - - -- Set FDW settings - FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p - LOOP - IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key) - THEN - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value); - ELSE - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value); - END IF; - END LOOP; - - -- Create specific role for this - IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN - EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name); - RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name; - END IF; - - -- Transfer ownership of the server to the fdw role - EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name); - - -- Create user mapping - -- NOTE: we use a PUBLIC user mapping but control access to the SERVER - -- so that we don't need to create a mapping for every user nor store credentials elsewhere - IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN - EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name); - RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name; - END IF; - - -- Update user mapping settings - FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP - IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value); - ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value); - END IF; - END LOOP; - - -- Grant usage on the wrapper and server to the fdw role - EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name); - RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name; - EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name); - RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name; - - -- Create schema if it does not exist. - IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN - EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name); - RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name; - END IF; - - -- Give the fdw role ownership over the schema - EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name); - RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name; - - -- TODO: Bring here the remote cdb_tablemetadata -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - --- A function to drop a user-defined foreign server and all related objects --- It does not read from CDB_Conf --- It must be executed with a superuser role to succeed --- --- Sample call: --- SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon') --- --- Note: if there's any dependent object (i.e. foreign table) this call will fail -CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false) -RETURNS void AS $$ -DECLARE - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); - cascade_clause NAME; -BEGIN - CASE force - WHEN true THEN - cascade_clause := 'CASCADE'; - ELSE - cascade_clause := 'RESTRICT'; - END CASE; - - EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Dropped schema %', fdw_objects_name; - EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name); - RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name; - EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Dropped foreign server %', fdw_objects_name; - EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name; - EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name); - RAISE NOTICE 'Dropped role %', fdw_objects_name; -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - --- Set up a user foreign table --- E.g: --- SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable'); --- SELECT * FROM amazon.my_table; -CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME) -RETURNS void AS $$ -DECLARE - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); -BEGIN - EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name); - --- Grant SELECT to fdw role - EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid) RETURNS TEXT AS $$ SELECT option_value FROM pg_options_to_table(( @@ -374,3 +204,12 @@ RETURNS timestamptz AS $$ LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid ) SELECT max(updated_at) FROM t_updated_at; $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Deprecated +-------------------------------------------------------------------------------- +DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name); +DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json); +DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean); +DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name); diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index 660d4e8..b29eb77 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -5,7 +5,6 @@ -- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }'); ---------------------------------- --- TODO: delete this development cleanup before final merge DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text); diff --git a/scripts-available/CDB_Organizations.sql b/scripts-available/CDB_Organizations.sql index c532ed5..94e228e 100644 --- a/scripts-available/CDB_Organizations.sql +++ b/scripts-available/CDB_Organizations.sql @@ -172,27 +172,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -------------------------------------------------------------------------------- --- Role management +-- Deprecated -------------------------------------------------------------------------------- -CREATE OR REPLACE -FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name) -RETURNS VOID AS $$ -DECLARE - org_role TEXT; -BEGIN - org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); - EXECUTE format('GRANT %I TO %I', role_name, org_role); -END -$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - -CREATE OR REPLACE -FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name) -RETURNS VOID AS $$ -DECLARE - org_role TEXT; -BEGIN - org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); - EXECUTE format('REVOKE %I FROM %I', role_name, org_role); -END -$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name); +DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name); diff --git a/sql/test_setup.sql b/sql/test_setup.sql index ec5021b..cf55bf4 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -2,7 +2,7 @@ \set QUIET on SET client_min_messages TO error; CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE SCHEMA cartodb; \i 'cartodb--unpackaged--@@VERSION@@.sql' CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 3e5658e..1963eaf 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -134,9 +134,9 @@ DROP TABLE t; -- table with single non-geometrical column CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT * FROM t; +SELECT cartodb_id, the_geom, description FROM t; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT * FROM t; +SELECT cartodb_id, the_geom, description FROM t; DROP TABLE t; -- table with existing srid-unconstrained (but type-constrained) the_geom diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 9939598..c5030a0 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -7,9 +7,9 @@ single non-geometrical column cartodbfied fine DROP TABLE SELECT 1 check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence DROP TABLE SELECT 1 srid-unconstrained the_geom cartodbfied fine diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql new file mode 100644 index 0000000..d314790 --- /dev/null +++ b/test/CDB_FederatedServer.sql @@ -0,0 +1,220 @@ +-- Setup +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +SET SESSION AUTHORIZATION postgres; +CREATE EXTENSION postgres_fdw; +\set QUIET off + +\echo '## List empty servers shows nothing' +SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## List non-existent server shows nothing' +SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server => 'doesNotExist'); + +\echo '## Create and list a server works' +SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## Create and list a second server works' +SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## List server by name works' +SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server => 'myRemote'); + + +\echo '## Re-register a second server works' +SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "other_remote_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## Unregister server 1 works' +SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote'::text); +SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## Unregistering a server that does not exist fails' +SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server => 'doesNotExist'::text); + +\echo '## Unregister the second server works' +SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote2'::text); +SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## Create a server with NULL name fails' +SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server => NULL::text, config => '{ "server": {}, "credentials" : {}}'); +\echo '## Create a server with NULL config fails' +SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => NULL::jsonb); +\echo '## Create a server with empty config fails' +SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{}'); +\echo '## Create a server without credentials fails' +SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + } +}'::jsonb); +\echo '## Create a server with empty credentials works' +SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { } +}'::jsonb); +SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); +SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server => 'empty'::text); +\echo '## Create a server without options fails' +SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ + "credentials": { + "username": "other_remote_user", + "password": "foobarino" + } +}'::jsonb); + +\echo '## Create a server with special characters works' +SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote" or''not'::text, config => '{ + "server": { + "dbname": "fdw target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "fdw user", + "password": "foo barino" + } +}'::jsonb); +SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers(); +SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote" or''not'::text); + +-- Test permissions +\set QUIET on +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; +\set QUIET off + +SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote3'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); + +\c contrib_regression cdb_fs_tester + +\echo '## All users are able to list servers' +SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); + +\echo '## Only superadmins can create servers' +SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); + + +\c contrib_regression postgres + +\echo '## Granting access to a user works' +SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT '9.55', cartodb.CDB_Federated_Server_List_Servers(); +\c contrib_regression postgres +SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name); +SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name); + +\echo '## Granting access again raises a notice' +SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); + +\echo '## Revoking access to a user works' +SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); +SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); + +\echo '## Unregistering a server with active grants works' +SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote3'::text); + + +\echo '## A user with granted access can not drop a server' +SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote4', db_role => 'cdb_fs_tester'::name); + +\c contrib_regression cdb_fs_tester +SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); + +\c contrib_regression postgres +SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); + + +-- Cleanup +\set QUIET on +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql new file mode 100644 index 0000000..da5e8cb --- /dev/null +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -0,0 +1,125 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +CREATE EXTENSION postgres_fdw; + +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; + +-- Create database to be used as remote +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ + "server": { + "host": "localhost", + "port": "12345" + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +SELECT 'C3', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback-no-port'::text, config => '{ + "server": { + "host": "localhost" + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +\c cdb_fs_tester postgres +CREATE EXTENSION postgis; +\c contrib_regression postgres +\set QUIET off + + +-- =================================================================== +-- Test server diagnostics function(s) +-- =================================================================== +\echo '%% It raises an error if the server does not exist' +SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist'); + +\echo '%% It returns a jsonb object' +SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')); + +\echo '%% It returns the server version' +SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb + FROM pg_settings WHERE name = 'server_version'; + +\echo '%% It returns the postgis version' +SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb + FROM pg_extension WHERE extname = 'postgis'; + +\echo '%% It returns null as the postgis version if it is not installed' +\set QUIET on +\c cdb_fs_tester postgres +DROP EXTENSION postgis; +\c contrib_regression postgres +\set QUIET off +SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb; + +\echo '%% It returns the remote server options' +SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; + +\echo '%% It returns network latency stats to the remote server: min <= avg <= max' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.1', (latency.ms->'min')::text::float <= (latency.ms->'avg')::text::float, (latency.ms->'avg')::text::float <= (latency.ms->'max')::text::float +FROM latency; + +\echo '%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.2', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 +FROM latency; + +\echo '%% Latency stats: stdev > 0' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0 +FROM latency; + +\echo '%% It raises an error if the wrong port is provided' +SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port'); + +\echo '%% Latency stats: can get them on default PG port 5432 when not provided' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms +) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 +FROM latency; + + +-- =================================================================== +-- Cleanup +-- =================================================================== +\set QUIET on +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text); +SELECT 'D3', cartodb.CDB_Federated_Server_Unregister(server => 'loopback-no-port'::text); +-- Reconnect, using a new session in order to close FDW connections +\connect +DROP DATABASE cdb_fs_tester; + +-- Drop role +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; + +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect new file mode 100644 index 0000000..044ad2f --- /dev/null +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -0,0 +1,28 @@ +C1| +C2| +C3| +%% It raises an error if the server does not exist +ERROR: Server "doesNotExist" does not exist +%% It returns a jsonb object +1.2|jsonb +%% It returns the server version +1.3|t +%% It returns the postgis version +1.4|t +%% It returns null as the postgis version if it is not installed +1.5|t +%% It returns the remote server options +1.6|t +%% It returns network latency stats to the remote server: min <= avg <= max +2.1|t|t +%% Latency stats: 0 <= min <= max <= 1000 ms (local connection) +2.2|t|t +%% Latency stats: stdev > 0 +2.3|t +%% It raises an error if the wrong port is provided +ERROR: could not connect to server "cdb_fs_wrong-port" +%% Latency stats: can get them on default PG port 5432 when not provided +2.4|t|t +D1| +D2| +D3| diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql new file mode 100644 index 0000000..8114673 --- /dev/null +++ b/test/CDB_FederatedServerListRemote.sql @@ -0,0 +1,319 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +CREATE EXTENSION postgres_fdw; + +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; +CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; + +-- Create database to be used as remote +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback2'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +-- =================================================================== +-- Setup 1 +-- =================================================================== +\c cdb_fs_tester cdb_fs_tester + +CREATE TYPE user_enum AS ENUM ('foo', 'bar', 'buz'); +CREATE SCHEMA "S 1"; +CREATE TABLE "S 1"."T 1" ( + "C 1" int NOT NULL, + c2 int NOT NULL, + c3 text, + c4 timestamptz, + c5 timestamp, + c6 varchar(10), + c7 char(10), + c8 user_enum, + CONSTRAINT t1_pkey PRIMARY KEY ("C 1") +); +CREATE TABLE "S 1"."T 2" ( + c1 int NOT NULL, + c2 text, + CONSTRAINT t2_pkey PRIMARY KEY (c1) +); +CREATE TABLE "S 1"."T 3" ( + c1 int NOT NULL, + c2 int NOT NULL, + c3 text, + CONSTRAINT t3_pkey PRIMARY KEY (c1) +); +CREATE TABLE "S 1"."T 4" ( + c1 int NOT NULL, + c2 int NOT NULL, + c3 text, + CONSTRAINT t4_pkey PRIMARY KEY (c1) +); + +-- Disable autovacuum for these tables to avoid unexpected effects of that +ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 3" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 4" SET (autovacuum_enabled = 'false'); + +\c contrib_regression postgres +SET client_min_messages TO notice; +\set VERBOSITY terse +\set QUIET off + + +-- =================================================================== +-- Test listing remote schemas +-- =================================================================== +\echo '## Test listing of remote schemas without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); + +\echo '## Test listing of remote schemas without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist'); + + +-- =================================================================== +-- Test listing remote tables +-- =================================================================== + +\echo '## Test listing of remote tables without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); + +\echo '## Test listing of remote tables without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1'); + +\echo '## Test listing of remote tables (rainy day): Remote schema does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist'); + + +-- =================================================================== +-- Test listing remote columns +-- =================================================================== + +\echo '## Test listing of remote columns without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); + +\echo '## Test listing of remote columns without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1'); + +\echo '## Test listing of remote columns (rainy day): Remote schema does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1'); + +\echo '## Test listing of remote columns (rainy day): Remote table does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist'); + +\echo '## Test listing of remote columns (rainy day): Remote table is NULL' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => NULL::text); + + +-- =================================================================== +-- Test that using a different user to list tables and dropping it +-- does not break the server: We use loopback2 as it's in a clean state +-- =================================================================== + + +\echo '## Test listing of remote objects with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester2'::name); +\c contrib_regression cdb_fs_tester2 +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); + +\c contrib_regression postgres +\echo '## Test that dropping the granted user works fine (sunny day)' +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; +DROP ROLE cdb_fs_tester2; + +\echo '## Test listing of remote objects with other user still works (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); + + +-- =================================================================== +-- Cleanup 1 +-- =================================================================== +\set QUIET on + +\c cdb_fs_tester cdb_fs_tester +DROP TABLE "S 1". "T 1"; +DROP TABLE "S 1". "T 2"; +DROP TABLE "S 1". "T 3"; +DROP TABLE "S 1". "T 4"; + +DROP SCHEMA "S 1"; +DROP TYPE user_enum; + + +-- =================================================================== +-- Setup 2: Using Postgis too +-- =================================================================== + +\c cdb_fs_tester postgres + +CREATE EXTENSION postgis; + +\c cdb_fs_tester cdb_fs_tester + +CREATE SCHEMA "S 1"; +CREATE TABLE "S 1"."T 5" ( + geom geometry(Geometry,4326), + geom_wm geometry(GeometryZ,3857), + geo_nosrid geometry, + geog geography +); + +\c contrib_regression postgres +SET client_min_messages TO notice; +\set VERBOSITY terse +\set QUIET off + + +-- =================================================================== +-- Test the listing functions +-- =================================================================== + +\echo '## Test listing of remote geometry columns (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); +\echo '## Test listing of remote geometry columns (sunny day) - Rerun' +-- Rerun should be ok +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); + + +-- =================================================================== +-- Test invalid password +-- =================================================================== + +\echo '## Check error message with invalid password (rainy day)' +SELECT cartodb.CDB_Federated_Server_Register_PG(server => 'loopback_invalid'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "wrong password" + } +}'::jsonb); + +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid'); + +SELECT cartodb.CDB_Federated_Server_Unregister(server => 'loopback_invalid'::text); + +-- =================================================================== +-- Cleanup 2 +-- =================================================================== +\set QUIET on + +\c cdb_fs_tester cdb_fs_tester +DROP TABLE "S 1". "T 5"; + +DROP SCHEMA "S 1"; + +\c contrib_regression postgres +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse + +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'loopback2'::text); + +DROP DATABASE cdb_fs_tester; + +-- Drop role +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; + +DROP EXTENSION postgres_fdw; + +\set QUIET off diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect new file mode 100644 index 0000000..244203c --- /dev/null +++ b/test/CDB_FederatedServerListRemote_expect @@ -0,0 +1,162 @@ +C1| +C2| +## Test listing of remote schemas without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas (sunny day) +S 1 +information_schema +public +## Test listing of remote schemas without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +S 1 +information_schema +public +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist +## Test listing of remote tables without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables (sunny day) +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +## Test listing of remote tables without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist +## Test listing of remote tables (rainy day): Remote schema does not exist +## Test listing of remote columns without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns (sunny day) +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +## Test listing of remote columns without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist +## Test listing of remote columns (rainy day): Remote schema does not exist +## Test listing of remote columns (rainy day): Remote table does not exist +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +## Test listing of remote columns (rainy day): Remote table is NULL +ERROR: Remote table name cannot be NULL +## Test listing of remote objects with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +S 1 +information_schema +public +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +You are now connected to database "contrib_regression" as user "postgres". +## Test that dropping the granted user works fine (sunny day) +REVOKE +DROP ROLE +## Test listing of remote objects with other user still works (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +S 1 +information_schema +public +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +## Test listing of remote geometry columns (sunny day) +geo_nosrid|GEOMETRY,0 +geog|Geometry,0 +geom|GEOMETRY,4326 +geom_wm|GEOMETRY,3857 +## Test listing of remote geometry columns (sunny day) - Rerun +geo_nosrid|GEOMETRY,0 +geog|Geometry,0 +geom|GEOMETRY,4326 +geom_wm|GEOMETRY,3857 +## Check error message with invalid password (rainy day) + +ERROR: could not connect to server "cdb_fs_loopback_invalid" + +D1| +D2| diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql new file mode 100644 index 0000000..0982398 --- /dev/null +++ b/test/CDB_FederatedServerTables.sql @@ -0,0 +1,405 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse + +CREATE EXTENSION postgres_fdw; + +-- We create a username following the same steps as organization members +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; + +CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; + +-- Create database to be used as remote +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + + +-- =================================================================== +-- create objects used through FDW loopback server +-- =================================================================== + +\c cdb_fs_tester postgres + +CREATE EXTENSION postgis; + +\c cdb_fs_tester cdb_fs_tester + +CREATE SCHEMA remote_schema; +CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326)); + +INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry); +INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry); + +CREATE TABLE remote_schema.remote_geom2(cartodb_id bigint, another_field text, the_geom geometry(Geometry,4326), the_geom_webmercator geometry(Geometry,3857)); + +INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)'); + +CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text); +INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes'); + +CREATE TABLE remote_schema.remote_geom3(id bigint, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); + +-- =================================================================== +-- Test the listing functions +-- =================================================================== + +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse +\set QUIET off + +\echo '## Registering an existing table works' +SELECT 'R1', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback.remote_geom; + +Select * FROM CDB_Federated_Server_List_Remote_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); + +\echo '## Registering another existing table works' +SELECT 'R2', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom2', + id_column => 'cartodb_id', + geom_column => 'the_geom', + webmercator_column => 'the_geom_webmercator', + local_name => 'myFullTable' + ); + +SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback."myFullTable"; +Select * FROM CDB_Federated_Server_List_Remote_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); + +\echo '## Re-registering a table works' +SELECT 'R3', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom2', + id_column => 'cartodb_id', + -- Switch geom and geom_column on purpose to force ST_Transform to be used + geom_column => 'the_geom_webmercator', + webmercator_column => 'the_geom', + local_name => 'different_name' + ); + +-- The old view should dissapear +SELECT 'S3_old', cartodb_id, another_field FROM cdb_fs_loopback."myFullTable"; +-- And the new appear +SELECT 'S3_new', cartodb_id, another_field FROM cdb_fs_loopback.different_name; + +\echo '## Unregistering works' +-- Deregistering the first table +SELECT 'U1', cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); +-- Selecting from the created view should fail now +SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; + +Select * FROM CDB_Federated_Server_List_Remote_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); + +-- =================================================================== +-- Test input +-- =================================================================== + +\echo '## Registering a table: Invalid server fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'Does not exist', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL server fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => NULL::text, + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid schema fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'Does not exist', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL schema fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => NULL::text, + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid table fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'Does not exist', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL table fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => NULL::text, + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid id fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'Does not exist', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL id fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => NULL::text, + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid geom_column fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'Does not exists' + ); + +\echo '## Registering a table: NULL geom_column is OK: Reuses geom_mercator' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => NULL::text, + webmercator_column => 'geom' + ); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; + +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + +\echo '## Registering a table: Invalid webmercator_column fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + webmercator_column => 'Does not exists' + ); + +\echo '## Registering a table: NULL webmercator_column is OK: Reuses geom' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + webmercator_column => NULL::text + ); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + +\echo '##Registering a table with extra columns show the correct information' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom3', + id_column => 'id', + geom_column => 'geom', + webmercator_column => 'geom_mercator' + ); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom3'; +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom3' + ); + +-- =================================================================== +-- Test conflicts +-- =================================================================== + +\echo '## Target conflict is handled nicely: Table' +CREATE TABLE cdb_fs_loopback.localtable (a integer); +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); + +\echo '## Target conflict is handled nicely: View' +CREATE VIEW cdb_fs_loopback.localtable2 AS Select * from cdb_fs_loopback.localtable; +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable2'); + +DROP VIEW cdb_fs_loopback.localtable2; +DROP TABLE cdb_fs_loopback.localtable; + +-- =================================================================== +-- Test permissions +-- =================================================================== + + +\echo '## Registering tables does not work without permissions' +\c contrib_regression cdb_fs_tester +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); + +\echo '## Normal users can not write in the import schema' +CREATE TABLE cdb_fs_loopback.localtable (a integer); + +\echo '## Listing remote tables does not work without permissions' +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); + +\echo '## Registering tables works with granted permissions' +\c contrib_regression postgres +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); + +\echo '## Listing remote tables works with granted permissions' +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); + +\echo '## Selecting from a registered table with granted permissions works' +Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; + +\echo '## Selecting from a registered table without permissions does not work' +\c contrib_regression cdb_fs_tester2 +CREATE OR REPLACE FUNCTION catch_permission_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN insufficient_privilege THEN + RETURN TRUE; + WHEN OTHERS THEN + RAISE WARNING 'Exception %', sqlstate; + RETURN FALSE; +END +$$ LANGUAGE 'plpgsql'; +Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable$$); +DROP FUNCTION catch_permission_error(text); + +\echo '## Deleting a registered table without permissions does not work' +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + +\echo '## Only the owner can grant permissions over the server' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); + +\echo '## Everything works for a different user when granted permissions' +\c contrib_regression postgres +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); +\c contrib_regression cdb_fs_tester2 +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; + +\echo '## A different user can unregister a table' +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); + +\echo '## Only the owner can revoke permissions over the server' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); + +-- =================================================================== +-- Cleanup +-- =================================================================== + +\set QUIET on +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse + +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; +DROP ROLE cdb_fs_tester2; + +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); +DROP DATABASE cdb_fs_tester; +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect new file mode 100644 index 0000000..6e02023 --- /dev/null +++ b/test/CDB_FederatedServerTables_expect @@ -0,0 +1,116 @@ +C1| +## Registering an existing table works +R1| +S1|1|POINT(1 1)|patata +S1|2|POINT(2 2)|patata2 +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +## Registering another existing table works +R2| +S2|3|POINT(3 3)|patata +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback."myFullTable"|cartodb_id|the_geom|the_geom_webmercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +## Re-registering a table works +R3| +ERROR: relation "cdb_fs_loopback.myFullTable" does not exist at character 49 +S3_new|3|patata +## Unregistering works +U1| +ERROR: relation "remote_geom" does not exist at character 71 +f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +## Registering a table: Invalid server fails +ERROR: Server "Does not exist" does not exist +## Registering a table: NULL server fails +ERROR: Server name cannot be NULL +## Registering a table: Invalid schema fails +ERROR: Could not import schema "Does not exist" of server "loopback": schema "Does not exist" is not present on foreign server "cdb_fs_loopback" +## Registering a table: NULL schema fails +ERROR: Schema name cannot be NULL +## Registering a table: Invalid table fails +ERROR: Could not import table "remote_schema.Does not exist" of server "loopback" +## Registering a table: NULL table fails +ERROR: Remote table name cannot be NULL +## Registering a table: Invalid id fails +ERROR: non integer id_column "Does not exist" +## Registering a table: NULL id fails +ERROR: non integer id_column "" +## Registering a table: Invalid geom_column fails +ERROR: non geometry column "Does not exists" +## Registering a table: NULL geom_column is OK: Reuses geom_mercator + +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] + +## Registering a table: Invalid webmercator_column fails +ERROR: non geometry column "Does not exists" +## Registering a table: NULL webmercator_column is OK: Reuses geom + +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] + +##Registering a table with extra columns show the correct information + +t|remote_geom3|cdb_fs_loopback.remote_geom3|id|geom|geom_mercator|[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] + +## Target conflict is handled nicely: Table +CREATE TABLE +ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists +## Target conflict is handled nicely: View +CREATE VIEW +ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable2" already exists +DROP VIEW +DROP TABLE +## Registering tables does not work without permissions +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +## Normal users can not write in the import schema +ERROR: permission denied for schema cdb_fs_loopback at character 14 +## Listing remote tables does not work without permissions +ERROR: Not enough permissions to access the server "loopback" +## Registering tables works with granted permissions +You are now connected to database "contrib_regression" as user "postgres". + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". + +## Listing remote tables works with granted permissions +t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +## Selecting from a registered table with granted permissions works +1|POINT(1 1) +2|POINT(2 2) +## Selecting from a registered table without permissions does not work +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +CREATE FUNCTION +t +DROP FUNCTION +## Deleting a registered table without permissions does not work +ERROR: Not enough permissions to access the server "loopback" +## Only the owner can grant permissions over the server +ERROR: You do not have rights to grant access on "loopback" +## Everything works for a different user when granted permissions +You are now connected to database "contrib_regression" as user "postgres". + +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +1|POINT(1 1) +2|POINT(2 2) +## A different user can unregister a table +NOTICE: drop cascades to view cdb_fs_loopback.localtable + +f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] +## Only the owner can revoke permissions over the server +ERROR: You do not have rights to revoke access on "loopback" +D1| diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect new file mode 100644 index 0000000..836646d --- /dev/null +++ b/test/CDB_FederatedServer_expect @@ -0,0 +1,69 @@ +## List empty servers shows nothing +## List non-existent server shows nothing +## Create and list a server works +1.3| +1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +## Create and list a second server works +2.1| +2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +## List server by name works +2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +## Re-register a second server works +3.1| +3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) +## Unregister server 1 works +4.1| +4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) +## Unregistering a server that does not exist fails +ERROR: Server "doesNotExist" does not exist +## Unregister the second server works +6.1| +## Create a server with NULL name fails +ERROR: Server name cannot be NULL +## Create a server with NULL config fails +7.01| +## Create a server with empty config fails +ERROR: Server information is mandatory +## Create a server without credentials fails +ERROR: Credentials are mandatory +## Create a server with empty credentials works +7.3| +7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,) +7.5| +## Create a server without options fails +ERROR: Server information is mandatory +## Create a server with special characters works +8.1| +8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") +8.3| +9.1| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +## All users are able to list servers +9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,) +## Only superadmins can create servers +ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw +You are now connected to database "contrib_regression" as user "postgres". +## Granting access to a user works +9.5| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +9.55|(myRemote3,postgres_fdw,localhost,5432,,read-only,fdw_user) +You are now connected to database "contrib_regression" as user "postgres". +ERROR: Server "does not exist" does not exist +ERROR: Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist +## Granting access again raises a notice +NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" +9.8| +## Revoking access to a user works +9.9| +9.10| +## Unregistering a server with active grants works +9.11| +## A user with granted access can not drop a server +10.1| +10.2| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to drop the server "myRemote4" +You are now connected to database "contrib_regression" as user "postgres". +10.4| diff --git a/test/CDB_SyncTableTest.sql b/test/CDB_SyncTableTest.sql index fc2711e..5b945ae 100644 --- a/test/CDB_SyncTableTest.sql +++ b/test/CDB_SyncTableTest.sql @@ -51,7 +51,7 @@ UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecodin SET client_min_messages TO notice; SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); SELECT * FROM test_sync_source ORDER BY cartodb_id; -SELECT * FROM test_sync_dest ORDER BY cartodb_id; +SELECT cartodb_id, the_geom, lat, lon, name FROM test_sync_dest ORDER BY cartodb_id; \echo 'It will work with schemas that need quoting' \set QUIET on diff --git a/test/CDB_SyncTableTest_expect b/test/CDB_SyncTableTest_expect index 01cb358..9e0c8df 100644 --- a/test/CDB_SyncTableTest_expect +++ b/test/CDB_SyncTableTest_expect @@ -54,10 +54,10 @@ NOTICE: MODIFIED 0 row(s) 2|||2|2|bar 4|||4|4|cantaloupe 5|||5|5|sandia -1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo -2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar -4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe -5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia +1|0101000020E6100000000000000000F03F000000000000F03F|1|1|foo +2|0101000020E610000000000000000000400000000000000040|2|2|bar +4|0101000020E610000000000000000010400000000000001040|4|4|cantaloupe +5|0101000020E610000000000000000014400000000000001440|5|5|sandia It will work with schemas that need quoting INSERT 0 1 diff --git a/test/CDB_Username_expect b/test/CDB_Username_expect index fe1ea5b..d030a42 100644 --- a/test/CDB_Username_expect +++ b/test/CDB_Username_expect @@ -1,4 +1,4 @@ -postgres +@@PGUSER@@ fulano fulanito diff --git a/test/extension/test.sh b/test/extension/test.sh index faf3d93..aab19f3 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -598,68 +598,6 @@ test_extension|public|"local-table-with-dashes"' sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;" sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't' - - # Check user-defined FDW's - # Set up a user foreign server - read -d '' ufdw_config <<- EOF -{ - "server": { - "extensions": "postgis", - "dbname": "fdw_target", - "host": "localhost", - "port": ${PGPORT:-5432} - }, - "user_mapping": { - "user": "fdw_user", - "password": "foobarino" - } -} -EOF - sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');" - - # Grant a user access to that FDW, and to grant to others - sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;' - - # Set up a user foreign table - sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');" - - # Check that the table can be accessed by the owner/creator - sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;' - sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - - # Check that a role with no permissions cannot use the FDW to access a remote table - sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails - - # Check that the table can be accessed by some other user by granting the role - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;' - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;' - - # Check that the table can be accessed by org members - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');" - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');" - - # By default publicuser cannot access the FDW - sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted - sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;' - - # If there are dependent objects, we cannot drop the foreign server - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails - sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;' - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" - - # But if there are, we can set the force flag to true to drop everything (defaults to false) - sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');" - sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;' - sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');" - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)" - - # Teardown DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;' From dfd26d930efcb6517c49fce5c0a70ba565c89e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 24 Dec 2019 00:52:04 +0100 Subject: [PATCH 5/6] Add huge warnings --- scripts-available/CDB_Organizations.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts-available/CDB_Organizations.sql b/scripts-available/CDB_Organizations.sql index 94e228e..68f6338 100644 --- a/scripts-available/CDB_Organizations.sql +++ b/scripts-available/CDB_Organizations.sql @@ -6,6 +6,13 @@ AS $$ $$ LANGUAGE SQL STABLE PARALLEL SAFE; + +----- ########################## WARNING ########################## +----- The code below creates a new role for the organization but +----- only when the extension is INSTALLED in a database, i.e. it +----- won't work if you clone a database that has it installed. +----- If you do, you need to update the extension to next and back +----- ########################## WARNING ########################## DO LANGUAGE 'plpgsql' $$ DECLARE cdb_org_member_role_name TEXT; @@ -38,6 +45,12 @@ AS $$ $$ LANGUAGE SQL STABLE PARALLEL SAFE; +----- ########################## WARNING ########################## +----- The code below creates a new role for the organization but +----- only when the extension is INSTALLED in a database, i.e. it +----- won't work if you clone a database that has it installed. +----- If you do, you need to update the extension to next and back +----- ########################## WARNING ########################## -- Administrator role creation on extension install DO LANGUAGE 'plpgsql' $$ DECLARE From 9fa532623dae4a76987f22f083e736eb65435158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 24 Dec 2019 13:56:46 +0100 Subject: [PATCH 6/6] Set python3 as dependency for PG12+ again --- .travis.yml | 3 --- Makefile | 4 +--- NEWS.md | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6f080c..f934592 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,6 @@ jobs: - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5" - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" - allow_failures: - - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" - - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" script: - sudo service postgresql stop; diff --git a/Makefile b/Makefile index 439e93e..d25f86c 100644 --- a/Makefile +++ b/Makefile @@ -140,9 +140,7 @@ PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) PLPYTHONU := plpythonu ifeq ($(PG_12_GE), true) -# Reverted until we are ready for PG12 support in other projects -PLPYTHONU := plpythonu -# PLPYTHONU := plpython3u +PLPYTHONU := plpython3u endif PGPORT ?= '5432' PGUSER ?= 'postgres' diff --git a/NEWS.md b/NEWS.md index ab968d0..fe2bd71 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 0.35.0 (XXXX-XX-XX) * Reapply the changes in 0.33.0 (the issue we were looking for was unrelated) +* Reapply `Make PG12 depend on plpython3u instead of plpythonu` 0.34.0 (2019-12-23) * Revert changes done in 0.33.0, keeping function signature to drop them