Check removal of SECURITY DEFINER

This is just a test to see how feasible is to remove the SECURITY
DEFINER and have regular users setup their FDW.

There are still problems with this approach:
- need to grant the usage of postgres_fdw (no big issue)
- need CREATEROLE privilege. A problem in itself (see the NOTES
https://www.postgresql.org/docs/current/sql-createrole.html)

Aside from those, there are still practical problems:
```
> Executing query 'SELECT cartodb.CDB_SetUp_User_Foreign_Server('test_user_fdw', '{
   "server": {
     "extensions": "postgis",
     "dbname": "fdw_target",
     "host": "localhost",
     "port": 5432
   },
   "user_mapping": {
     "user": "fdw_user",
     "password": "foobarino"
   }
}');' as cdb_testmember_1
ERROR:  permission denied for foreign-data wrapper postgres_fdw
CONTEXT:  SQL statement "ALTER SERVER test_user_fdw OWNER TO test_user_fdw"
PL/pgSQL function cdb_setup_user_foreign_server(name,json) line 32 at EXECUTE
```

.travis.yml

@@ -1,3 +1,4 @@
+dist: xenial
 language: c
 sudo: required
 
@@ -6,29 +7,27 @@ env:
     - PGUSER=postgres
     - PGDATABASE=postgres
     - PGOPTIONS='-c client_min_messages=NOTICE'
+    - PGPORT=5432
+    - POSTGIS_VERSION="2.5"
 
-jobs:
-  include:
-    - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
-      dist: xenial
-    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
-      dist: bionic
-    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
-      dist: bionic
-    - env: POSTGRESQL_VERSION="13" POSTGIS_VERSION="3"
-      dist: bionic
+  matrix:
+    - POSTGRESQL_VERSION="9.6"
+    - POSTGRESQL_VERSION="10"
+    - POSTGRESQL_VERSION="11"
 
-script:
+
+before_install:
+  - sudo service postgresql stop;
   - sudo apt-get remove postgresql* -y
   - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
   - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
-  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
-  # For pre12, install plpython2. For PG12 install plpython3
-  - if [[ $POSTGRESQL_VERSION == '11' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-$POSTGRESQL_VERSION python3 python3-redis; fi;
-  - for i in $(pg_lsclusters  | tail -n +2 | awk '{print $1}'); do sudo pg_dropcluster --stop $i main; done;
-  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION
-  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -- --auth-local trust --auth-host password
-  - export PGPORT=$(pg_lsclusters | grep $POSTGRESQL_VERSION | awk '{print $3}')
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
+  - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
+  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password
+  - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
+  - sudo pip install redis==2.4.9
+script:
   - make
   - sudo make install
   - make installcheck
@@ -37,5 +36,3 @@ after_failure:
   - pg_lsclusters
   - cat regression.out
   - cat regression.diffs
-  - echo $PGPORT
-  - sudo cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.37.1
+EXTVERSION = 0.28.1
 
 SED = sed
 AWK = awk
@@ -101,15 +101,6 @@ UPGRADABLE = \
   0.27.2 \
   0.28.0 \
   0.28.1 \
-  0.29.0 \
-  0.30.0 \
-  0.31.0 \
-  0.32.0 \
-  0.33.0 \
-  0.34.0 \
-  0.35.0 \
-  0.36.0 \
-  0.37.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -120,6 +111,8 @@ UPGRADES = \
      $(SED) 's/$$/--$(EXTVERSION).sql/' | \
      $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g')
 
+GITDIR=$(shell test -d .git && echo '.git' || cat .git | $(SED) 's/^gitdir: //')
+
 DATA_built = \
   $(EXTENSION)--$(EXTVERSION).sql \
   $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql \
@@ -131,27 +124,17 @@ EXTRA_CLEAN = cartodb_version.sql
 DOCS = README.md
 REGRESS_OLD = $(wildcard test/*.sql)
 REGRESS_LEGACY = $(REGRESS_OLD:.sql=)
-REGRESS = test/test_setup $(REGRESS_LEGACY)
+REGRESS = test_setup $(REGRESS_LEGACY)
 
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
-PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
-PLPYTHONU := plpythonu
-ifeq ($(PG_12_GE), true)
-PLPYTHONU := plpython3u
-endif
-PGPORT ?= '5432'
-PGUSER ?= 'postgres'
-
 $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
 	$(SED) 	-e 's/@extschema@/cartodb/g' \
-		-e 's/@postgisschema@/public/g' \
-		-e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
+		-e "s/@postgisschema@/public/g" >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 
@@ -165,10 +148,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 	cp $< $@
 
 $(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
-cartodb_version.sql: cartodb_version.sql.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
+cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" $< > $@
 
 # Needed for consistent `echo` results with backslashes
 SHELL = bash
@@ -177,10 +160,6 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
 	mkdir -p expected/test/
 	mkdir -p results/test/
-	cat sql/test_setup.sql | \
-			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-			> sql/test/test_setup.sql
-	cp sql/test_setup_expect expected/test/test_setup.out
 	for f in $(REGRESS_OLD); do \
 		tn=`basename $${f} .sql`; \
 		of=sql/test/$${tn}.sql; \
@@ -189,23 +168,10 @@ legacy_regress: $(REGRESS_OLD) Makefile
 		echo '\t' >> $${of}; \
 		echo '\set QUIET off' >> $${of}; \
 		cat $${f} | \
-			$(SED) 	-e 's/@@VERSION@@/$(EXTVERSION)/' \
-				-e 's/@extschema@/cartodb/g' \
-				-e "s/@postgisschema@/public/g" \
-				-e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-				-e 's/@@PGPORT@@/$(PGPORT)/g' \
-				-e 's/@@PGUSER@@/$(PGUSER)/g' \
-				>> $${of}; \
+			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \
 		exp=expected/test/$${tn}.out; \
 		echo '\set ECHO none' > $${exp}; \
-		cat test/$${tn}_expect | \
-			$(SED) 	-e 's/@@VERSION@@/$(EXTVERSION)/' \
-				-e 's/@extschema@/cartodb/g' \
-				-e "s/@postgisschema@/public/g" \
-				-e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-				-e 's/@@PGPORT@@/$(PGPORT)/g' \
-				-e 's/@@PGUSER@@/$(PGUSER)/g' \
-				>> $${exp}; \
+		cat test/$${tn}_expect >> $${exp}; \
 	done
 
 test_organization:
@@ -214,15 +180,7 @@ test_organization:
 test_extension_new:
 	bash test/extension/test.sh
 
-legacy_tests: legacy_regress $(EXTENSION)--unpackaged--$(EXTVERSION).sql
+legacy_tests: legacy_regress
 
-PGREGRESS := $(shell dirname `$(PG_CONFIG) --pgxs`)/../../src/test/regress/pg_regress
-PGBINDIR := $(shell $(PG_CONFIG) --bindir)
-PGREGRESSDATABASE = 'contrib_regression'
-regress: legacy_tests
-	PGUSER=$(PGUSER) \
-	PGPORT=$(PGPORT) \
-	$(PGREGRESS) --inputdir=./ --bindir='$(PGBINDIR)' --dbname=$(PGREGRESSDATABASE) $(REGRESS)
+installcheck: legacy_tests test_extension_new test_organization
 
-installcheck: test_extension_new test_organization
-	$(MAKE) -C . regress

NEWS.md

@@ -1,50 +1,3 @@
-0.37.1 (2020-12-02)
-* Change `__CDB_RegenerateTable_Get_Commands` to use the caller timeout or '1min' if not set.
-
-0.37.0 (2020-11-26)
-* Raised minimum PG version to 11.
-* Add `CDB_RegenerateTable` function to regenerate a table.
-* Add `CDB_GetTableQueries` to get the queries of a table (constraints, indices, triggers...).
-* Add `CDB_ApplyQueriesSafe` to apply the queries of `CDB_GetTableQueries` discarding any exceptions.
-* Deprecate creation of new overview tables.
-* _cdb_has_usable_geom_record: Check only the extension schema.
-
-0.36.0 (2020-02-13)
-* Make `_CDB_Group_API_Auth` python3 compatible by passing bytes representation instead of a string.
-* Make `_CDB_Group_API_Request` python3 compatible by adapting the function signature of `HTTPConnection`.
-
-0.35.0 (2019-12-30)
-* Reapply the changes in 0.33.0 (the issue we were looking for was unrelated)
-* Reapply `Make PG12 depend on plpython3u instead of plpythonu`
-* Fix identifier quotation in `CDB_UserDataSize`
-
-0.34.0 (2019-12-23)
-* Revert changes done in 0.33.0, keeping function signature to drop them
-
-0.33.0 (2019-12-20)
-* Revert `Make PG12 depend on plpython3u instead of plpythonu`.
-* Add functions to manage Federated Tables (Foreign Data Wrapper)
-
-0.32.0 (2019-11-08)
-* Fix oAuth ownership re-assignation for functions
-* Some fixes for PG12.
-* Make PG12 depend on plpython3u instead of plpythonu
-* CDB_UserDataSize is now compatible with postgis 3 without postgis_raster.
-* Makefile: Add regress target (checks regress tests without needing to install the extension)
-
-0.31.0 (2019-10-08)
-* Ghost tables: Add missing tags (#370)
-* Set search_path in security definer functions.
-
-0.30.0 (2019-07-17)
-* Added new admin functions to connect CARTO with user FDW's (#369)
-
-0.29.0 (2019-07-15)
-* Added new function CDB_OAuth:
-  * Install event trigger to check for table/view/sequence/function creation
-  * Reassign the ownership of new objects to a defined role in the cdb_conf
-* Changed MakeFile to support different expects for differents PG versions
-
 0.28.1 (2019-07-04)
 * Avoid temporary tables creation in CDB_SyncTable (#366)
 * Make CDB_Get_Foreign_Updated_At robust to missing CDB_TableMetadata (#362)

README.md

@@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/
 Dependencies
 ------------
 
- * PostgreSQL 11+ (with plpythonu extension). For PostgreSQL 12+ plpython3u is required instead. Older versions might still work but they aren't actively tested or supported.
+ * PostgreSQL 9.6+ (with plpythonu extension and xml support)
  * [PostGIS extension](http://postgis.net)
  * Python with [Redis module](https://pypi.org/project/redis/)
 
@@ -30,7 +30,7 @@ make installcheck
 
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
-
+      
 NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
@@ -39,7 +39,23 @@ Enable database
 In a database that needs to be turned into a "cartodb" user database, run:
 
 ```sql
-CREATE EXTENSION cartodb CASCADE;
+CREATE EXTENSION postgis;
+CREATE EXTENSION cartodb;
+```
+
+Migrate existing cartodb database
+---------------------------------
+
+When upgrading an existing cartodb user database, the cartodb extension
+can be migrated from the "unpackaged" version. The procedure will copy
+the data from ``public.CDB_TableMetada`` to ``cartodb.CDB_TableMetadata``,
+re-cartodbfy all tables using old functions in triggers and drop the
+cartodb functions from the 'public' schema. All new cartodb objects will
+be in the "cartodb" schema.
+
+```sql
+CREATE EXTENSION postgis FROM unpackaged;
+CREATE EXTENSION cartodb FROM unpackaged;
 ```
 
 Update cartodb extension

cartodb.control.in

@@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
 superuser = true
 relocatable = false
 schema = cartodb
-requires = '@@plpythonu@@, postgis'
+requires = 'plpythonu, postgis'

doc/CDB_Overviews.md

@@ -1,8 +1,3 @@
-
-============================ WARNING ===================================
-Creating overviews is no longer supported.
-============================ WARNING ===================================
-
 Overviews are tables that represent a *reduced* version of a dataset intended
 for efficient rendering at certain zoom levels while preserving the
 general visual appearance of the complete dataset.

expected/test_setup.out

@@ -1,5 +1,6 @@
-SET client_min_messages TO error;
-CREATE EXTENSION cartodb CASCADE;
+CREATE EXTENSION postgis;
+CREATE EXTENSION plpythonu;
+CREATE EXTENSION cartodb;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN

scripts-available/CDB_AnalysisCheck.sql

@@ -6,11 +6,8 @@ $$
 BEGIN
   RETURN @extschema@.CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
 END;
-$$  LANGUAGE 'plpgsql'
-    STABLE
-    PARALLEL SAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$
+LANGUAGE 'plpgsql' STABLE PARALLEL SAFE SECURITY DEFINER;
 
 
 -- Get the factor (fraction of the quota) for Camshaft cached analysis tables

scripts-available/CDB_CartodbfyTable.sql

@@ -636,7 +636,8 @@ $$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE;
 
 DO $$
 BEGIN
-  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record' AND typnamespace = '@extschema@'::regnamespace) THEN
+  SET search_path TO @extschema@;
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record') THEN
     CREATE TYPE @extschema@._cdb_has_usable_geom_record
       AS (has_usable_geoms boolean,
         text_geom_column boolean,
@@ -845,7 +846,7 @@ BEGIN
   -- values. No usable pk implies has_usable_pk_sequence = false.
   has_usable_pk_sequence := false;
   IF has_usable_primary_key THEN
-    SELECT @extschema@._CDB_Has_Usable_PK_Sequence(reloid)
+    SELECT _CDB_Has_Usable_PK_Sequence(reloid)
     INTO STRICT has_usable_pk_sequence;
   END IF;
 

scripts-available/CDB_EstimateRowCount.sql

@@ -12,12 +12,7 @@ BEGIN
     EXECUTE Format('ANALYZE %s;', reloid);
   END IF;
 END
-$$  LANGUAGE 'plpgsql'
-    VOLATILE
-    STRICT
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Return a row count estimate of the result of a query using statistics
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EstimateRowCount(query text)

scripts-available/CDB_FederatedServer.sql

@@ -1,437 +0,0 @@
---------------------------------------------------------------------------------
--- Private functions
---------------------------------------------------------------------------------
-
---
--- This function is just a placement to store and use the pattern for
--- foreign object names
--- Servers:         cdb_fs_$(server_name)
--- View schema:     cdb_fs_$(server_name)
---  > This is where all views created when importing tables are placed
---  > One server has only one view schema
--- Import Schemas:  cdb_fs_schema_$(md5sum(server_name || remote_schema_name))
---  > This is where the foreign tables are placed
---  > One server has one import schema per remote schema plus auxiliar ones used
---      to access the remote catalog (pg_catalog, information_schema...)
--- Owner role:      cdb_fs_$(md5sum(current_database() || server_name)
---  > This is the role than owns all schemas and tables related to the server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern()
-RETURNS TEXT
-AS $$
-    SELECT 'cdb_fs_'::text;
-$$
-LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
-
---
--- Produce a valid DB name for servers generated for the Federated Server
--- If check_existence is true, it'll throw if the server doesn't exists
--- This name is also used as the schema to store views
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL)
-RETURNS NAME
-AS $$
-DECLARE
-    internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name);
-BEGIN
-    IF input_name IS NULL OR char_length(input_name) = 0 THEN
-        RAISE EXCEPTION 'Server name cannot be NULL';
-    END IF;
-
-    -- We discard anything that would be truncated
-    IF (char_length(internal_server_name) >= 64) THEN
-        RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name;
-    END IF;
-
-    IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN
-        RAISE EXCEPTION 'Server "%" does not exist', input_name;
-    END IF;
-
-    RETURN internal_server_name::name;
-END
-$$
-LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-
---
--- Given the internal name for a remote server, it returns the name used by the user
--- Reverses __CDB_FS_Generate_Server_Name
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME)
-RETURNS TEXT
-AS $$
-    SELECT right(internal_server_name,
-            char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT;
-$$
-LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
-
---
--- Produce a valid name for a schema generated for the Federated Server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT)
-RETURNS NAME
-AS $$
-DECLARE
-    hash_value text := md5(internal_server_name::text || '__' || schema_name::text);
-BEGIN
-    IF schema_name IS NULL THEN
-        RAISE EXCEPTION 'Schema name cannot be NULL';
-    END IF;
-    RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name;
-END
-$$
-LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-
---
--- Produce a valid name for a role generated for the Federated Server
--- This needs to include the current database in its hash to avoid collisions in clusters with more than one database
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME)
-RETURNS NAME
-AS $$
-DECLARE
-    hash_value text := md5(current_database()::text || '__' || internal_server_name::text);
-    role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value);
-BEGIN
-    RETURN role_name::name;
-END
-$$
-LANGUAGE PLPGSQL STABLE PARALLEL SAFE;
-
---
--- Creates (if not exist) a schema to place the objects for a remote schema
--- The schema is with the same AUTHORIZATION as the server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT)
-RETURNS NAME
-AS $$
-DECLARE
-    schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name);
-    role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
-BEGIN
-    -- By changing the local role to the owner of the server we have an
-    -- easy way to check for permissions and keep all objects under the same owner
-    BEGIN
-        EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
-    EXCEPTION
-    WHEN invalid_parameter_value THEN
-        RAISE EXCEPTION 'Server "%" does not exist',
-                        @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
-    WHEN OTHERS THEN
-        RAISE EXCEPTION 'Not enough permissions to access the server "%"',
-                        @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
-    END;
-
-    IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN
-        EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name);
-    END IF;
-    RETURN schema_name;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- Returns the type of a server by internal name
--- Currently all of them should be postgres_fdw
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME)
-RETURNS name
-AS $$
-    SELECT f.fdwname
-        FROM pg_foreign_server s
-        JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid
-        WHERE s.srvname = internal_server_name;
-$$
-LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-
---
--- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server
--- 
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB)
-RETURNS jsonb
-AS $$
-DECLARE
-    mapping jsonb := '{}'::jsonb;
-BEGIN
-    IF NOT (input_config ? 'credentials') THEN
-        RAISE EXCEPTION 'Credentials are mandatory';
-    END IF;
-
-    -- For now, allow not passing username or password
-    IF input_config->'credentials'->'username' IS NOT NULL THEN
-        mapping := jsonb_build_object('user', input_config->'credentials'->'username');
-    END IF;
-    IF input_config->'credentials'->'password' IS NOT NULL THEN
-        mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password');
-    END IF;
-
-    RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping);
-END
-$$
-LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-
--- Take a config jsonb as input and return it augmented with default
--- options
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb)
-RETURNS jsonb
-AS $$
-DECLARE
-    default_options jsonb := '{
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    }';
-    server_config jsonb;
-BEGIN
-    IF NOT (input_config ? 'server') THEN
-        RAISE EXCEPTION 'Server information is mandatory';
-    END IF;
-    server_config := default_options || to_jsonb(input_config->'server');
-    RETURN jsonb_set(input_config, '{server}'::text[], server_config);
-END
-$$
-LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-
--- Given an server name, returns the username used in the configuration if the caller has rights to access it
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME)
-RETURNS text
-AS $$
-DECLARE
-    role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
-    username text;
-BEGIN
-    BEGIN
-        EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
-    EXCEPTION WHEN OTHERS THEN
-        RETURN NULL;
-    END;
-
-    SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username
-        FROM pg_foreign_server s
-        LEFT JOIN pg_user_mappings u
-        ON u.srvid = s.oid
-        WHERE s.srvname = internal_server_name
-        ORDER BY 1;
-
-    RESET ROLE;
-
-    RETURN username;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---------------------------------------------------------------------------------
--- Public functions
---------------------------------------------------------------------------------
-
-
---
--- Registers a new PG server
---
--- Example config: '{
---     "server": {
---         "dbname": "fdw_target",
---         "host": "localhost",
---         "port": 5432,
---         "extensions": "postgis",
---         "updatable": "false",
---         "use_remote_estimate": "true",
---         "fetch_size": "1000"
---     },
---     "credentials": {
---         "username": "fdw_user",
---         "password": "foobarino"
---     }
--- }'
---
--- The configuration from __CDB_FS_add_default_options will be appended
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
-    final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config));
-    role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
-    row record;
-    option record;
-BEGIN
-    IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
-        RAISE EXCEPTION 'postgres_fdw extension is not installed'
-            USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`';
-    END IF;
-
-    -- We only create server and roles if the server didn't exist before
-    IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN
-        BEGIN
-            EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal);
-            IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN
-                EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name);
-            END IF;
-            EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name);
-
-            -- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases
-            -- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database
-            EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name);
-            EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name);
-            EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name);
-            EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name);
-            EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name);
-
-            EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
-            EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
-            EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name);
-            EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name);
-            EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal);
-        EXCEPTION WHEN OTHERS THEN
-            RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM
-                USING HINT = 'Please clean the left over objects';
-        END;
-    END IF;
-
-    -- Add new options
-    FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p
-    LOOP
-        IF NOT EXISTS (
-            WITH a AS (
-                SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal
-            ) SELECT * from a where options = row.key)
-        THEN
-            EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value);
-        ELSE
-            EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value);
-        END IF;
-    END LOOP;
-
-    -- Update user mapping settings
-    FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o
-    LOOP
-        IF NOT EXISTS (
-            WITH a AS (
-                SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name
-            ) SELECT * from a where options = option.key)
-        THEN
-            EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value);
-        ELSE
-            EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value);
-        END IF;
-    END LOOP;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- Drops a registered server and all the objects associated with it
--- 
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
-BEGIN
-    SET client_min_messages = ERROR;
-    BEGIN
-        EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal);
-        EXECUTE FORMAT ('DROP OWNED BY %I', role_name);
-        EXECUTE FORMAT ('DROP ROLE %I', role_name);
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server;
-    END;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- List registered servers
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%')
-RETURNS TABLE (
-    name        text,
-    driver      text,
-    host        text,
-    port        text,
-    dbname      text,
-    readmode    text,
-    username    text
-)
-AS $$
-DECLARE
-    server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server);
-BEGIN
-    RETURN QUERY SELECT 
-        -- Name as shown to the user
-        @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name",
-
-        -- Which driver are we using (postgres_fdw, odbc_fdw...)
-        @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver",
-
-        -- Read options from pg_foreign_server
-        (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host",
-        (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port",
-        (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName",
-        CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode",
-
-        @extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username"
-    FROM pg_foreign_server s
-    LEFT JOIN pg_user_mappings u
-    ON u.srvid = s.oid
-    WHERE s.srvname ILIKE server_name
-    ORDER BY 1;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE;
-
-
---
--- Grant access to a server
--- In the future we might consider adding the server's view schema to the role search_path
--- to make it easier to access the created views
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
-BEGIN
-    IF (db_role IS NULL) THEN
-        RAISE EXCEPTION 'User role "%" cannot be NULL', username;
-    END IF;
-    BEGIN
-        EXECUTE format('GRANT %I TO %I', server_role_name, db_role);
-    EXCEPTION
-    WHEN insufficient_privilege THEN
-        RAISE EXCEPTION 'You do not have rights to grant access on "%"', server;
-    WHEN OTHERS THEN
-        RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM;
-    END;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- Revoke access to a server
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
-BEGIN
-    IF (db_role IS NULL) THEN
-        RAISE EXCEPTION 'User role "%" cannot be NULL', username;
-    END IF;
-    BEGIN
-        EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role);
-    EXCEPTION
-    WHEN insufficient_privilege THEN
-        RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server;
-    WHEN OTHERS THEN
-        RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM;
-    END;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_FederatedServerDiagnostics.sql

@@ -1,243 +0,0 @@
---------------------------------------------------------------------------------
--- Private functions
---------------------------------------------------------------------------------
-
---
--- Import a foreign table if it does not exist
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name)
-RETURNS void
-AS $$
-DECLARE
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-BEGIN
-    IF NOT EXISTS (
-        SELECT * FROM pg_class
-        WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
-        AND relname = remote_table
-    ) THEN
-        EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
-                    remote_schema, remote_table, server_internal, local_schema);
-    END IF;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- Get the version of a remote PG server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name)
-RETURNS text
-AS $$
-DECLARE
-    remote_schema name := 'pg_catalog';
-    remote_table name := 'pg_settings';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-    remote_server_version text;
-BEGIN
-    PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
-
-    BEGIN
-        EXECUTE format('
-            SELECT setting FROM %I.%I WHERE name = ''server_version'';
-        ', local_schema, remote_table) INTO remote_server_version;
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Not enough permissions to access the server "%"',
-                        @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
-    END;
-
-    RETURN remote_server_version;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---
--- Get the PostGIS extension version of a remote PG server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name)
-RETURNS text
-AS $$
-DECLARE
-    remote_schema name := 'pg_catalog';
-    remote_table name := 'pg_extension';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-    remote_postgis_version text;
-BEGIN
-    PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
-
-    BEGIN
-        EXECUTE format('
-            SELECT extversion FROM %I.%I WHERE extname = ''postgis'';
-        ', local_schema, remote_table) INTO remote_postgis_version;
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Not enough permissions to access the server "%"',
-                        @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
-    END;
-
-    RETURN remote_postgis_version;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---
--- Get the foreign server options of a remote PG server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name)
-RETURNS jsonb
-AS $$
-    -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html
-    -- See https://www.postgresql.org/docs/current/functions-info.html
-    SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM (
-        SELECT (pg_options_to_table(srvoptions)).*
-        FROM pg_foreign_server
-        WHERE srvname = server_internal
-    ) AS opt;
-$$
-LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-
-
---
--- Get a foreign PG server hostname from the catalog
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name)
-RETURNS text
-AS $$
-    SELECT option_value FROM (
-        SELECT (pg_options_to_table(srvoptions)).*
-        FROM pg_foreign_server WHERE srvname = server_internal
-    ) AS opt WHERE opt.option_name = 'host';
-$$
-LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-
-
---
--- Get a foreign PG server port from the catalog
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name)
-RETURNS integer
-AS $$
-    SELECT option_value::integer FROM (
-        SELECT (pg_options_to_table(srvoptions)).*
-        FROM pg_foreign_server WHERE srvname = server_internal
-    ) AS opt WHERE opt.option_name = 'port';
-$$
-LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-
---
--- Get one measure of network latency in ms to a remote TCP server
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(
-    server_internal name,
-    timeout_seconds float DEFAULT 5.0,
-    n_samples integer DEFAULT 10
-)
-RETURNS jsonb
-AS $$
-    import socket
-    import json
-    import math
-    from timeit import default_timer as timer
-
-    plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name'])
-    rv = plpy.execute(plan, [server_internal], 1)
-    host = rv[0]['host']
-
-    plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name'])
-    rv = plpy.execute(plan, [server_internal], 1)
-    port = rv[0]['port'] or 5432
-
-    n_errors = 0
-    samples = []
-
-    for i in range(n_samples):
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        s.settimeout(timeout_seconds)
-
-        t_start = timer()
-
-        try:
-            s.connect((host, int(port)))
-            t_stop = timer()
-            s.shutdown(socket.SHUT_RD)
-        except (socket.timeout, OSError, socket.error) as ex:
-            plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex)))
-            n_errors += 1
-        finally:
-            s.close()
-
-        t_connect = (t_stop - t_start) * 1000.0
-        plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect))
-        samples.append(t_connect)
-
-    stats = {
-        'n_samples': n_samples,
-        'n_errors': n_errors,
-    }
-    n = len(samples)
-    if n >= 1:
-        mean = sum(samples) / n
-        stats.update({
-            'avg': round(mean, 3),
-            'min': round(min(samples), 3),
-            'max': round(max(samples), 3)
-        })
-    if n >= 2:
-        var = sum([ (x - mean)**2 for x in samples ]) / (n-1)
-        stdev = math.sqrt(var)
-        stats.update({
-            'stdev': round(stdev, 3)
-        })
-    return json.dumps(stats)
-$$
-LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
-
-
---
--- Collect and return diagnostics info from a remote PG into a jsonb
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name)
-RETURNS jsonb
-AS $$
-DECLARE
-    remote_server_version  text    := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal);
-    remote_postgis_version text    := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal);
-    remote_server_options jsonb    := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal);
-    remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal);
-BEGIN
-    RETURN jsonb_build_object(
-        'server_version', remote_server_version,
-        'postgis_version', remote_postgis_version,
-        'server_options', remote_server_options,
-        'server_latency_ms', remote_server_latency_ms
-    );
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
-
---------------------------------------------------------------------------------
--- Public functions
---------------------------------------------------------------------------------
-
---
--- Collect and return diagnostics info from a remote PG into a jsonb
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT)
-RETURNS jsonb
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_type name := @extschema@.__CDB_FS_server_type(server_internal);
-BEGIN
-    CASE server_type
-    WHEN 'postgres_fdw' THEN
-        RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal);
-    ELSE
-        RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
-    END CASE;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_FederatedServerListRemote.sql

@@ -1,298 +0,0 @@
---------------------------------------------------------------------------------
--- Private functions
---------------------------------------------------------------------------------
-
---
--- List the schemas of a remote PG server
--- 
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name)
-RETURNS TABLE(remote_schema name)
-AS $$
-DECLARE
-    -- Import schemata from the information schema
-    --
-    -- "The view schemata contains all schemas in the current database
-    -- that the current user has access to (by way of being the owner
-    -- or having some privilege)."
-    -- See https://www.postgresql.org/docs/11/infoschema-schemata.html
-    --
-    -- "The information schema is defined in the SQL standard and can
-    -- therefore be expected to be portable and remain stable"
-    -- See https://www.postgresql.org/docs/11/information-schema.html
-    inf_schema name := 'information_schema';
-    remote_table name := 'schemata';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
-BEGIN
-    -- Import the foreign schemata table
-    IF NOT EXISTS (
-        SELECT * FROM pg_class
-        WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
-        AND relname = remote_table
-    ) THEN
-        EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
-                    inf_schema, remote_table, server_internal, local_schema);
-    END IF;
-
-    -- Return the result we're interested in. Exclude toast and temp schemas
-    BEGIN
-        RETURN QUERY EXECUTE format('
-            SELECT schema_name::name AS remote_schema FROM %I.%I
-            WHERE schema_name NOT LIKE %s
-            ORDER BY remote_schema
-        ', local_schema, remote_table, '''pg_%''');
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Not enough permissions to access the server "%"',
-                        @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
-    END;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- List the names of the tables in a remote PG schema
--- 
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name)
-RETURNS TABLE(remote_table name)
-AS $func$
-DECLARE
-    -- Import `tables` from the information schema
-    --
-    -- "The view tables contains all tables and views defined in the
-    -- current database. Only those tables and views are shown that
-    -- the current user has access to (by way of being the owner or
-    -- having some privilege)."
-    -- https://www.postgresql.org/docs/11/infoschema-tables.html
-
-    -- Create local target schema if it does not exists
-    inf_schema name := 'information_schema';
-    remote_table name := 'tables';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
-BEGIN
-    -- Import the foreign `tables` if not done
-    IF NOT EXISTS (
-        SELECT * FROM pg_class
-        WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
-        AND relname = remote_table
-    ) THEN
-        EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
-                    inf_schema, remote_table, server_internal, local_schema);
-    END IF;
-
-    -- Note: in this context, schema names are not to be quoted
-    RETURN QUERY EXECUTE format($q$
-        SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name
-        $q$, local_schema, remote_table, remote_schema);
-END
-$func$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---
--- List the columns in a remote PG schema
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name)
-RETURNS TABLE(table_name name, column_name name, column_type text)
-AS $func$
-DECLARE
-    -- Import `columns` from the information schema
-    --
-    -- "The view columns contains information about all table columns (or view columns)
-    -- in the database. System columns (oid, etc.) are not included. Only those columns 
-    -- are shown that the current user has access to (by way of being the owner or having some privilege)."
-    -- https://www.postgresql.org/docs/11/infoschema-columns.html
-
-    -- Create local target schema if it does not exists
-    inf_schema name := 'information_schema';
-    remote_col_table name := 'columns';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
-BEGIN
-    -- Import the foreign `columns` if not done
-    IF NOT EXISTS (
-        SELECT * FROM pg_class
-        WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
-        AND relname = remote_col_table
-    ) THEN
-        EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
-                    inf_schema, remote_col_table, server_internal, local_schema);
-    END IF;
-
-    -- Note: in this context, schema names are not to be quoted
-    -- We join with the geometry columns to change the type `USER-DEFINED` 
-    -- by its appropiate geometry and srid
-    RETURN QUERY EXECUTE format($q$
-        SELECT 
-            a.table_name::name,
-            a.column_name::name,
-            COALESCE(b.column_type, a.data_type)::TEXT as column_type
-        FROM
-            %I.%I a
-        LEFT JOIN
-            @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b
-        ON a.table_name = b.table_name AND a.column_name = b.column_name
-        WHERE table_schema = '%s'
-        ORDER BY a.table_name, a.column_name $q$,
-        local_schema, remote_col_table,
-        server_internal, remote_schema,
-        remote_schema);
-END
-$func$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- List the geometry columns in a remote PG schema
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public')
-RETURNS TABLE(table_name name, column_name name, column_type text)
-AS $func$
-DECLARE
-    -- Import `geometry_columns` and `geography_columns` from the postgis schema
-    -- We assume that postgis is installed in the public schema
-
-    -- Create local target schema if it does not exists
-    remote_geometry_view name := 'geometry_columns';
-    remote_geography_view name := 'geography_columns';
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema);
-BEGIN
-    -- Import the foreign `geometry_columns` and `geography_columns` if not done
-    IF NOT EXISTS (
-        SELECT * FROM pg_class
-        WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
-        AND relname = remote_geometry_view
-    ) THEN
-        EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I',
-                    postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema);
-    END IF;
-
-    BEGIN
-    -- Note: We return both the type and srid as the type
-        RETURN QUERY EXECUTE format($q$
-            SELECT 
-                f_table_name::NAME as table_name,
-                f_geometry_column::NAME as column_name,
-                type::TEXT || ',' || srid::TEXT as column_type
-            FROM
-            (
-                SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I
-            ) _geo_views
-            WHERE f_table_schema = '%s'
-        $q$,
-            local_schema, remote_geometry_view,
-            local_schema, remote_geography_view,
-            remote_schema);
-    EXCEPTION WHEN OTHERS THEN
-        RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"',
-                    postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
-        RETURN;
-    END;
-END
-$func$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---------------------------------------------------------------------------------
--- Public functions
---------------------------------------------------------------------------------
-
---
--- List remote schemas in a federated server that the current user has access to.
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT)
-RETURNS TABLE(remote_schema name)
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_type name := @extschema@.__CDB_FS_server_type(server_internal);
-BEGIN
-    CASE server_type
-    WHEN 'postgres_fdw' THEN
-        RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal);
-    ELSE
-        RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
-    END CASE;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- List remote tables in a federated server that the current user has access to.
--- For registered tables it returns also the associated configuration
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT)
-RETURNS TABLE(
-    registered boolean,
-    remote_table TEXT,
-    local_qualified_name TEXT,
-    id_column_name TEXT,
-    geom_column_name TEXT,
-    webmercator_column_name TEXT,
-    columns JSON
-    )
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_type name := @extschema@.__CDB_FS_server_type(server_internal);
-BEGIN
-    CASE server_type
-    WHEN 'postgres_fdw' THEN
-        RETURN QUERY
-            SELECT
-                coalesce(registered_tables.registered, false)::boolean as registered,
-                foreign_tables.remote_table::text as remote_table,
-                registered_tables.local_qualified_name as local_qualified_name,
-                registered_tables.id_column_name as id_column_name,
-                registered_tables.geom_column_name as geom_column_name,
-                registered_tables.webmercator_column_name as webmercator_column_name,
-                remote_columns.columns as columns
-            FROM
-                @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables
-            LEFT JOIN
-                @extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables
-            ON foreign_tables.remote_table = registered_tables.remote_table
-            LEFT JOIN
-                (   -- Extract and group columns with their remote table
-                    SELECT  table_name,
-                            json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns
-                    FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
-                    GROUP BY table_name
-                ) remote_columns
-            ON foreign_tables.remote_table = remote_columns.table_name
-            ORDER BY foreign_tables.remote_table;
-    ELSE
-        RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
-    END CASE;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- List the columns of a remote table in a federated server that the current user has access to.
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns(
-    server TEXT,
-    remote_schema TEXT,
-    remote_table TEXT)
-RETURNS TABLE(column_n name, column_t text)
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
-    server_type name := @extschema@.__CDB_FS_server_type(server_internal);
-BEGIN
-    IF remote_table IS NULL THEN
-        RAISE EXCEPTION 'Remote table name cannot be NULL';
-    END IF;
-
-    CASE server_type
-    WHEN 'postgres_fdw' THEN
-        RETURN QUERY
-        SELECT 
-            column_name,
-            column_type
-        FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
-        WHERE table_name = remote_table
-        ORDER BY column_name;
-    ELSE
-        RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
-    END CASE;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_FederatedServerTables.sql

@@ -1,345 +0,0 @@
---------------------------------------------------------------------------------
--- Private functions
---------------------------------------------------------------------------------
-
---
--- Checks if a column is of integer type
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME)
-RETURNS boolean
-AS $$
-BEGIN
-    PERFORM atttypid FROM pg_catalog.pg_attribute
-       WHERE attrelid = input_table
-         AND attname = colname
-         AND atttypid IN (SELECT oid FROM pg_type
-           WHERE typname IN
-             ('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8'));
-    RETURN FOUND;
-END
-$$
-LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
-
---
--- Checks if a column is of geometry type
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME)
-RETURNS boolean
-AS $$
-BEGIN
-    PERFORM atttypid FROM pg_catalog.pg_attribute
-        WHERE attrelid = input_table
-           AND attname = colname
-           AND atttypid = '@postgisschema@.geometry'::regtype;
-    RETURN FOUND;
-END
-$$
-LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
-
---
--- Returns the name of all the columns from a table
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS)
-RETURNS SETOF NAME
-AS $$
-    SELECT
-        a.attname as "colname"
-    FROM pg_catalog.pg_attribute a
-    WHERE
-        a.attnum > 0
-            AND NOT a.attisdropped
-            AND a.attrelid = (
-                SELECT c.oid
-                FROM pg_catalog.pg_class c
-                LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-                WHERE c.oid = input_table::oid
-            )
-    ORDER BY a.attnum;
-$$ LANGUAGE SQL STABLE PARALLEL UNSAFE;
-
---
--- Returns the id column from a view definition
--- Note: The id is always of one of this forms:
--- SELECT t.cartodb_id,
--- SELECT t.my_id as cartodb_id,
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT)
-RETURNS TEXT
-AS $$
-    WITH column_definitions AS
-    (
-        SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
-    )
-    SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2))
-    FROM column_definitions
-    WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id')
-    LIMIT 1;
-$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
-
---
--- Returns the geom column from a view definition
---
--- Note: The the_geom is always of one of this forms:
---      t.the_geom,
---      t.my_geom as the_geom,
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT)
-RETURNS TEXT
-AS $$
-    WITH column_definitions AS
-    (
-        SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
-    )
-    SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
-    FROM column_definitions
-    WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom')
-    LIMIT 1;
-$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
-
---
--- Returns the webmercatorcolumn from a view definition
--- Note: The the_geom_webmercator is always of one of this forms:
---      t.the_geom_webmercator,
---      t.my_geom as the_geom_webmercator,
--- Or without the trailing comma:
---      t.the_geom_webmercator
---      t.my_geom as the_geom_webmercator
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT)
-RETURNS TEXT
-AS $$
-    WITH column_definitions AS
-    (
-        SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
-    )
-    SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
-    FROM column_definitions
-    WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator')
-    LIMIT 1;
-$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
-
-
---
--- List all registered tables in a server + schema
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables(
-    server_internal NAME,
-    remote_schema TEXT
-    )
-RETURNS TABLE(
-    registered boolean,
-    remote_table TEXT,
-    local_qualified_name TEXT,
-    id_column_name TEXT,
-    geom_column_name TEXT,
-    webmercator_column_name TEXT
-    )
-AS $$
-DECLARE
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-BEGIN
-    RETURN QUERY SELECT
-        true as registered,
-        source_table::text as remote_table,
-        format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name,
-        @extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name,
-        @extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name,
-        @extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name
-    FROM
-    (
-        SELECT DISTINCT
-            dependent_ns.nspname as dependent_schema,
-            dependent_view.relname as dependent_view,
-            source_table.relname as source_table,
-            pg_get_viewdef(dependent_view.oid) as view_definition
-        FROM pg_depend
-        JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
-        JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
-        JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
-        JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace
-        JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
-        WHERE
-        source_ns.nspname = local_schema
-        ORDER BY 1,2
-    ) _aux;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---------------------------------------------------------------------------------
--- Public functions
---------------------------------------------------------------------------------
-
---
--- Sets up a Federated Table
---
--- Precondition: the federated server has to be set up via
--- CDB_Federated_Server_Register_PG
---
--- Postcondition: it generates a view in the schema of the user that
--- can be used through SQL and Maps API's.
--- If the table was already exported, it will be dropped and re-imported
---
--- The view is placed under the server's view schema (cdb_fs_$(server_name))
---
--- E.g:
--- SELECT cartodb.CDB_SetUp_PG_Federated_Table(
---   'amazon',                  -- mandatory, name of the federated server
---   'my_remote_schema',        -- mandatory, schema name
---   'my_remote_table',         -- mandatory, table name
---   'id',                      -- mandatory, name of the id column
---   'geom',                    -- optional, name of the geom column, preferably in 4326
---   'webmercator'              -- optional, should be in 3857 if present
---   'local_name'               -- optional, name of the local view (uses the remote_name if not declared)
--- );
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register(
-    server TEXT,
-    remote_schema TEXT,
-    remote_table TEXT,
-    id_column TEXT,
-    geom_column TEXT DEFAULT NULL,
-    webmercator_column TEXT DEFAULT NULL,
-    local_name NAME DEFAULT NULL
-)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-    role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
-
-    src_table REGCLASS; -- import_schema.remote_table  - import_schema is local_schema
-    local_view text;    -- view_schema.local_name      - view_schema is server_internal
-
-    rest_of_cols TEXT[];
-    geom_expression TEXT;
-    webmercator_expression TEXT;
-    carto_columns_expression TEXT[];
-BEGIN
-    IF remote_table IS NULL THEN
-        RAISE EXCEPTION 'Remote table name cannot be NULL';
-    END IF;
-
-    -- Make do with whatever columns are provided
-    IF webmercator_column IS NULL THEN
-        webmercator_column := geom_column;
-    ELSIF geom_column IS NULL THEN
-        geom_column := webmercator_column;
-    END IF;
-
-    IF local_name IS NULL THEN
-        local_name := remote_table;
-    END IF;
-
-    -- Import the foreign table
-    -- Drop the old view / table if there was one
-    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN
-        EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table);
-    END IF;
-    BEGIN
-        EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;',
-                        remote_schema, remote_table, server_internal, local_schema);
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM;
-    END;
-
-    BEGIN
-        src_table := format('%I.%I', local_schema, remote_table);
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server;
-    END;
-
-    -- Check id_column is numeric
-    IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN
-        RAISE EXCEPTION 'non integer id_column "%"', id_column;
-    END IF;
-
-    -- Check if the geom and mercator columns have a geometry type (if provided)
-    IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN
-        RAISE EXCEPTION 'non geometry column "%"', geom_column;
-    END IF;
-    IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN
-        RAISE EXCEPTION 'non geometry column "%"', webmercator_column;
-    END IF;
-
-    -- Get a list of columns excluding the id, geom and the_geom_webmercator
-    SELECT ARRAY(
-        SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c
-        WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL)
-    ) INTO rest_of_cols;
-
-    IF geom_column IS NULL
-    THEN
-        geom_expression := 'NULL AS the_geom';
-    ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326
-    THEN
-        geom_expression := format('t.%I AS the_geom', geom_column);
-    ELSE
-        -- It needs an ST_Transform to 4326
-        geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column);
-    END IF;
-
-    IF webmercator_column IS NULL
-    THEN
-        webmercator_expression := 'NULL AS the_geom_webmercator';
-    ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857
-    THEN
-        webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column);
-    ELSE
-        -- It needs an ST_Transform to 3857
-        webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column);
-    END IF;
-
-    -- CARTO columns expressions
-    carto_columns_expression := ARRAY[
-        format('t.%1$I AS cartodb_id', id_column),
-        geom_expression,
-        webmercator_expression
-    ];
-
-    -- Create view schema if it doesn't exist
-    IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN
-        EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name);
-    END IF;
-
-    -- Create a view with homogeneous CDB fields
-    BEGIN
-        EXECUTE format(
-            'CREATE OR REPLACE VIEW %1$I.%2$I AS
-                SELECT %3s
-                FROM %4$s t',
-            server_internal, local_name,
-            array_to_string(carto_columns_expression || rest_of_cols, ','),
-            src_table
-        );
-    EXCEPTION WHEN OTHERS THEN
-        IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN
-            RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name;
-        ELSE
-            RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM;
-        END IF;
-    END;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
---
--- Unregisters a remote table. Any dependent object will be dropped
---
-CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister(
-    server TEXT,
-    remote_schema TEXT,
-    remote_table TEXT
-)
-RETURNS void
-AS $$
-DECLARE
-    server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
-    local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
-BEGIN
-    EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table);
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_ForeignTable.sql

@@ -139,6 +139,117 @@ $$
 LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
 
 
+-- A function to set up a user-defined foreign data server
+-- It does not read from CDB_Conf
+--
+-- Sample call:
+-- SELECT cartodb.CDB_SetUp_User_Foreign_Server('amazon', '{
+--    "server": {
+--      "extensions": "postgis",
+--      "dbname": "testdb",
+--      "host": "myhostname.us-east-2.rds.amazonaws.com",
+--      "port": "5432"
+--    },
+--    "user_mapping": {
+--      "user": "fdw_user",
+--      "password": "secret"
+--    }
+-- }');
+--
+-- Underneath it will:
+--   * Set up postgresql_fdw
+--   * Create a server with the name 'amazon'
+--   * Create a role called 'amazon' to manage access
+--   * Create a user mapping with that role 'amazon'
+--   * Create a schema 'amazon' as a convenience to set up all foreign
+--     tables over there
+--
+-- It is the responsibility of the caller to grant that role to either:
+--   * Nobody
+--   * Specific roles: GRANT amazon TO role_name;
+--   * Members of the organization: SELECT cartodb.CDB_Grant_Role_To_Org_Members('amazon'); TODO
+--   * The publicuser: GRANT amazon TO publicuser;
+CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_Foreign_Server(fdw_name NAME, config json)
+RETURNS void AS $$
+DECLARE
+  row record;
+  option record;
+BEGIN
+  -- Create FDW first if it does not exist
+  IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
+    THEN
+    EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
+  END IF;
+
+  -- Set FDW settings
+  FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
+    LOOP
+      IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
+        THEN
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
+      ELSE
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
+      END IF;
+    END LOOP;
+
+    -- Create specific role for this
+    IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_name) THEN
+       EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_name);
+    END IF;
+
+    -- Grant the fdw role to the caller, and permissions to grant it to others
+    EXECUTE FORMAT ('GRANT %I TO %I WITH ADMIN OPTION', fdw_name, session_user);
+
+    -- Transfer ownership of the server to the fdw role
+    EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_name, fdw_name);
+
+    -- Create user mapping
+    -- NOTE: we use a PUBLIC user mapping but control access to the SERVER
+    -- so that we don't need to create a mapping for every user nor store credentials elsewhere
+    IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = 'public' ) THEN
+        EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_name);
+    END IF;
+
+    -- Update user mapping settings
+    FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP
+        IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = 'public') SELECT * from a where options = option.key) THEN
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_name, option.key, option.value);
+        ELSE
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_name, option.key, option.value);
+        END IF;
+    END LOOP;
+
+    -- Grant usage on the wrapper and server to the fdw role
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_name);
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_name, fdw_name);
+
+    -- Create schema if it does not exist.
+    IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
+      EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
+    END IF;
+
+    -- Give the fdw role ownership over the schema
+    EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_name, fdw_name);
+
+    -- TODO: Bring here the remote cdb_tablemetadata
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
+-- Set up a user foreign table
+-- E.g:
+--   SELECT cartodb.CDB_SetUp_User_Foreign_Table('amazon', 'carto_lite', 'mytable');
+--   SELECT * FROM amazon.my_table;
+CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_Foreign_Table(fdw_name NAME, foreign_schema NAME, table_name NAME)
+RETURNS void AS $$
+BEGIN
+  EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_name, fdw_name);
+  --- Grant SELECT to fdw role
+  EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_name, table_name, fdw_name);
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
 CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid)
 RETURNS TEXT AS $$
     SELECT option_value FROM pg_options_to_table((
@@ -204,12 +315,3 @@ RETURNS timestamptz AS $$
         LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
     ) SELECT max(updated_at) FROM t_updated_at;
 $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-
-
---------------------------------------------------------------------------------
--- Deprecated
---------------------------------------------------------------------------------
-DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name);
-DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json);
-DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean);
-DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name);

scripts-available/CDB_GhostTables.sql

@@ -32,6 +32,7 @@ AS $$
           client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
           GD['invalidation'] = client
         except Exception as err:
+          error = "client_error - %s" % str(err)
           # NOTE: no retries on connection error
           plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' +  str(err))
           break
@@ -40,12 +41,13 @@ AS $$
       client.execute_command('DBSCH', db_name, username, event_name)
       break
     except Exception as err:
+      error = "request_error - %s" % str(err)
       client = GD['invalidation'] = None # force reconnect
       if not tis_retry:
         plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' +  str(err))
         break
       tis_retry -= 1 # try reconnecting
-$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
 
 -- Enqueues a job to run Ghost tables linking process for the current user
 CREATE OR REPLACE FUNCTION @extschema@.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
@@ -59,13 +61,9 @@ AS $$
     EXECUTE 'SELECT current_database();' INTO db_name;
 
     PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
-    RAISE INFO '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
   END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Trigger function to call CDB_LinkGhostTables()
 CREATE OR REPLACE FUNCTION @extschema@._CDB_LinkGhostTablesTrigger()
@@ -78,11 +76,7 @@ AS $$
     PERFORM @extschema@.CDB_LinkGhostTables(ddl_tag);
     RETURN NULL;
   END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Event trigger to save the current transaction in @extschema@.cdb_ddl_execution
 CREATE OR REPLACE FUNCTION @extschema@.CDB_SaveDDLTransaction()
@@ -91,11 +85,7 @@ AS $$
   BEGIN
     INSERT INTO @extschema@.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT ON CONSTRAINT cdb_ddl_execution_pkey DO NOTHING;
   END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Creates the trigger on DDL events to link ghost tables
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableGhostTablesTrigger()
@@ -116,33 +106,7 @@ AS $$
 
     CREATE EVENT TRIGGER link_ghost_tables
     ON ddl_command_end
-    WHEN TAG IN ('CREATE TABLE',
-                'SELECT INTO',
-                'DROP TABLE',
-                'ALTER TABLE',
-
-                'CREATE TRIGGER',
-                'DROP TRIGGER',
-                'ALTER TRIGGER',
-
-                'CREATE VIEW',
-                'DROP VIEW',
-                'ALTER VIEW',
-
-                'CREATE FOREIGN TABLE',
-                'ALTER FOREIGN TABLE',
-                'DROP FOREIGN TABLE',
-
-                'ALTER MATERIALIZED VIEW',
-                'CREATE MATERIALIZED VIEW',
-                'DROP MATERIALIZED VIEW',
-
-                'IMPORT FOREIGN SCHEMA',
-
-                'DROP EXTENSION',
-                'DROP SCHEMA',
-                'DROP SERVER',
-                'DROP TYPE')
+    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW', 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE', 'DROP FOREIGN TABLE')
     EXECUTE PROCEDURE @extschema@.CDB_SaveDDLTransaction();
   END;
 $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Groups_API.sql

@@ -5,6 +5,7 @@
 -- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
 ----------------------------------
 
+-- TODO: delete this development cleanup before final merge
 DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text);
 DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text);
 DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text);
@@ -21,91 +22,59 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text)
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
 
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(old_group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
     body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -120,20 +89,13 @@ FUNCTION @extschema@._CDB_Group_Table_GrantPermission_API(group_name text, usern
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
     body = '{ "access": "%s" }' % access
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -148,19 +110,12 @@ FUNCTION @extschema@._CDB_Group_Table_RevokeAllPermission_API(group_name text, u
     RETURNS VOID AS
 $$
     import string
-    try:
-        from urllib import pathname2url
-    except:
-        from urllib.request import pathname2url
+    import urllib
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
     plpy.execute(query)
-$$  LANGUAGE '@@plpythonu@@'
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -191,36 +146,22 @@ $$
       params = json.loads(conf)
       auth = 'Basic %s' % plpy.execute("SELECT @extschema@._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
       return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
-$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
     RETURNS TEXT AS
 $$
-    import sys
     import base64
-
-    data_to_encode = '%s:%s' % (username, password)
-    if sys.version_info[0] < 3:
-      data_encoded = base64.encodestring(data_to_encode)
-    else:
-      data_encoded = base64.b64encode(data_to_encode.encode()).decode()
-
-    data_encoded = data_encoded.replace('\n', '')
-    return data_encoded
-$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
 
 -- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
     RETURNS int AS
 $$
-    python_v2 = True
-    try:
-      import httplib as client
-    except:
-      from http import client
-      python_v2 = False
+    import httplib
 
     params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0]
     if params['host'] is None:
@@ -233,25 +174,22 @@ $$
     last_err = None
     while retry > 0:
       try:
-        if python_v2:
-          conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout'])
-        else:
-          conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], port=params['port'], timeout=params['timeout'])
+        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
         database_name = plpy.execute("select current_database();")[0]['current_database']
-        conn.request(method, url.format(database_name), body, headers)
-        response = conn.getresponse()
+        client.request(method, url.format(database_name), body, headers)
+        response = client.getresponse()
         assert response.status in valid_return_codes
         return response.status
       except Exception as err:
         retry -= 1
         last_err = err
         plpy.warning('Retrying after: ' + str(err))
-        conn = SD['groups_api_client'] = None
+        client = SD['groups_api_client'] = None
 
     if last_err is not None:
       plpy.error('Fatal Group API error: ' + str(last_err))
       raise last_err
 
     return None
-$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
 revoke all on function @extschema@._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_OAuth.sql

@@ -1,65 +0,0 @@
--- Function that reassign the owner of a table to their ownership_role
-CREATE OR REPLACE FUNCTION @extschema@.CDB_OAuthReassignTableOwnerOnCreation()
-  RETURNS event_trigger
-  AS $$
-DECLARE
-    obj record;
-    owner_role text;
-    creator_role text;
-BEGIN
-    FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
-    LOOP
-      RAISE DEBUG '% ddl object: % % % %',
-                  tg_tag,
-                  obj.command_tag,
-                  obj.object_type,
-                  obj.schema_name,
-                  obj.object_identity;
-      IF obj.object_type = 'function' THEN
-        SELECT rolname FROM pg_proc JOIN pg_roles ON proowner = pg_roles.oid WHERE pg_proc.oid = obj.objid INTO creator_role;
-      ELSE
-	      SELECT rolname FROM pg_class JOIN pg_roles ON relowner = pg_roles.oid WHERE pg_class.oid = obj.objid INTO creator_role;
-      END IF;
-      SELECT value->>'ownership_role_name' from @extschema@.CDB_Conf_GetConf('api_keys_' || quote_ident(creator_role)) value INTO owner_role;
-      IF owner_role IS NULL OR owner_role = '' THEN
-        RAISE DEBUG 'owner_role not found';
-        CONTINUE;
-      ELSE
-        EXECUTE 'ALTER ' || obj.object_type || ' ' || obj.object_identity || ' OWNER TO ' || quote_ident(owner_role);
-        IF obj.object_type = 'function' THEN
-          EXECUTE 'GRANT ALL ON FUNCTION ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
-        ELSE
-          EXECUTE 'GRANT ALL ON ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
-        END IF;        
-        RAISE DEBUG 'Changing ownership from % to %', creator_role, owner_role;
-      END IF;
-    END LOOP;
-END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
-
--- Creates the trigger on DDL events in order to reassign the owner
-CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableOAuthReassignTablesTrigger()
-RETURNS void
-AS $$
-  BEGIN
-    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
-
-    CREATE EVENT TRIGGER oauth_reassign_tables_trigger
-    ON ddl_command_end
-    WHEN TAG IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'CREATE VIEW', 'CREATE FOREIGN TABLE', 'CREATE MATERIALIZED VIEW', 'CREATE SEQUENCE', 'CREATE FUNCTION')
-    EXECUTE PROCEDURE @extschema@.CDB_OAuthReassignTableOwnerOnCreation();
-  END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-
--- Deletes the trigger on DDL events in order to reassign the owner
-CREATE OR REPLACE FUNCTION @extschema@.CDB_DisableOAuthReassignTablesTrigger()
-RETURNS void
-AS $$
-  BEGIN
-    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
-  END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Organizations.sql

@@ -6,13 +6,6 @@ AS $$
 $$
 LANGUAGE SQL STABLE PARALLEL SAFE;
 
-
------ ########################## WARNING ##########################
------ The code below creates a new role for the organization but
------ only when the extension is INSTALLED in a database, i.e. it
------ won't work if you clone a database that has it installed.
------ If you do, you need to update the extension to next and back
------ ########################## WARNING ##########################
 DO LANGUAGE 'plpgsql' $$
 DECLARE
     cdb_org_member_role_name TEXT;
@@ -45,12 +38,6 @@ AS $$
 $$
 LANGUAGE SQL STABLE PARALLEL SAFE;
 
------ ########################## WARNING ##########################
------ The code below creates a new role for the organization but
------ only when the extension is INSTALLED in a database, i.e. it
------ won't work if you clone a database that has it installed.
------ If you do, you need to update the extension to next and back
------ ########################## WARNING ##########################
 -- Administrator role creation on extension install
 DO LANGUAGE 'plpgsql' $$
 DECLARE
@@ -182,10 +169,3 @@ BEGIN
     EXECUTE 'SELECT @extschema@.CDB_Organization_Remove_Access_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || @extschema@.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-
---------------------------------------------------------------------------------
--- Deprecated
---------------------------------------------------------------------------------
-DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name);
-DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name);

scripts-available/CDB_Overviews.sql

@@ -268,7 +268,44 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
+    fraction := power(2, 2*(overview_z - ref_z));
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Sampling reduce stategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Sampling reduce stategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Estimate number of rows
+    SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
+      WHERE oid = reloid::oid;
+
+    IF class_info.relpages < 2 OR fraction > 0.5 THEN
+      -- We'll avoid possible CDB_RandomTids problems
+      EXECUTE Format('
+        %s SELECT * FROM %s WHERE random() < %s;
+      ', creation_clause, reloid, fraction);
+    ELSE
+      num_samples := ceil(class_info.reltuples*fraction);
+      EXECUTE Format('
+        %1$s SELECT * FROM %2$s
+          WHERE ctid = ANY (
+            ARRAY[
+              (SELECT @extschema@.CDB_RandomTids(''%2$s'', %3$s))
+            ]
+          );
+      ', creation_clause, reloid, num_samples);
+    END IF;
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -281,7 +318,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 -- This function is declared SECURITY DEFINER so it executes with the privileges
 -- of the function creator to have a chance to alter the privileges of the
 -- overview table to match those of the dataset. It will only perform any change
--- if the overview table belongs to the same scheme as the dataset and it
+-- if the overview table belgons to the same scheme as the dataset and it
 -- matches the scheme naming for overview tables.
 CREATE OR REPLACE FUNCTION @extschema@._CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
 RETURNS VOID
@@ -294,13 +331,38 @@ AS $$
     overview_scheme TEXT;
     overview_name TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    -- This function will only register a table as an overview table if it matches
+    -- the overviews naming scheme for the dataset and z level and the table belongs
+    -- to the same scheme as the the dataset
+    SELECT * FROM @extschema@._cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name;
+    SELECT * FROM @extschema@._cdb_split_table_name(overview_table) INTO overview_scheme, overview_name;
+    IF dataset_scheme = overview_scheme AND
+       overview_name = @extschema@._CDB_OverviewTableName(dataset_name, overview_z) THEN
+
+      -- preserve the owner of the base table
+      SELECT u.usename
+        FROM pg_catalog.pg_class c
+          JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
+          JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = dataset_name::text AND n.nspname = dataset_scheme
+        INTO table_owner;
+
+      EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
+
+      -- preserve the table privileges
+      UPDATE pg_class c_to
+        SET  relacl = c_from.relacl
+        FROM  pg_class c_from
+        WHERE c_from.oid  = dataset
+        AND   c_to.oid    = overview_table;
+
+      PERFORM @extschema@._CDB_Add_Indexes(overview_table);
+
+      -- TODO: If metadata about existing overviews is to be stored
+      -- it should be done here (CDB_Overviews would consume such metadata)
+    END IF;
   END
-$$  LANGUAGE PLPGSQL
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Dataset attributes (column names other than the
 -- CartoDB primary key and geometry columns) which should be aggregated
@@ -331,7 +393,7 @@ DECLARE
   attr_list TEXT;
 BEGIN
   SELECT string_agg(s.c, ',') FROM (
-    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
+    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -488,7 +550,7 @@ DECLARE
 BEGIN
   SELECT string_agg(@extschema@._CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
   FROM (
-    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
+    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -551,7 +613,105 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    SELECT @extschema@._CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT @extschema@.CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := @extschema@._CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := @extschema@._CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        WHEN 'the_geom' THEN
+          Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom)
+        WHEN 'the_geom_webmercator' THEN
+           Format('%s AS the_geom_webmercator', point_geom)
+        ELSE c
+        END AS column
+        FROM @extschema@.CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and selecte the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+      %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             Floor(@postgisschema@.ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(@postgisschema@.ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s f
+          WHERE f.the_geom_webmercator IS NOT NULL
+          GROUP BY gx, gy
+         )
+         SELECT %6$s FROM clusters
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -636,7 +796,7 @@ AS $$
           '@postgisschema@.ST_Transform(@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom'
         WHEN 'the_geom_webmercator' THEN
           '@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator'
-        ELSE c::text
+        ELSE c
         END AS column
         FROM CDB_ColumnNames(reloid) c
     )
@@ -708,7 +868,104 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    SELECT @extschema@._CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT @extschema@.CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := @extschema@._CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := @extschema@._CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        ELSE c
+        END AS column
+        FROM @extschema@.CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster sampling strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster sampling strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and select the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+       %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             Floor(@postgisschema@.ST_X(_f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(@postgisschema@.ST_Y(_f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s _f
+          GROUP BY gx, gy
+         ),
+         cluster_geom AS (
+           SELECT the_geom, the_geom_webmercator, clusters.*
+             FROM clusters INNER JOIN %1$s _g ON (clusters.cartodb_id = _g.cartodb_id)
+         )
+         SELECT %6$s FROM cluster_geom
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -729,7 +986,9 @@ AS $$
 DECLARE
   tolerance_px FLOAT8;
 BEGIN
-  RAISE EXCEPTION 'Creating overviews is deprecated';
+  -- Use the default tolerance
+  tolerance_px := 1.0;
+  RETURN @extschema@.CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
 END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -748,7 +1007,54 @@ DECLARE
   has_counter_column boolean;
   has_overviews_for_z boolean;
 BEGIN
-  RAISE EXCEPTION 'Creating overviews is deprecated';
+  -- Determine the referece zoom level
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
+
+  IF ref_z < 0 OR ref_z IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  -- Determine overlay zoom levels
+  -- TODO: should be handled by the refscale_strategy?
+  overview_z := ref_z - 1;
+  WHILE overview_z >= 0 LOOP
+    SELECT array_append(overviews_z, overview_z) INTO overviews_z;
+    overview_z := overview_z - overviews_step;
+  END LOOP;
+
+  --  TODO Check for non-used overview to delete them but we have to be aware that the
+  --       current queries, for example from a tiler, are been used with the old overviews
+  --       so if we remove the overviews in the process this could lead to errors
+
+  -- Create or reganerate overlay tables
+  base_z := ref_z;
+  base_rel := reloid;
+  FOREACH overview_z IN ARRAY overviews_z LOOP
+    SELECT CASE WHEN count(*) > 0 THEN TRUE ELSE FALSE END from CDB_Overviews(reloid) WHERE z = overview_z INTO has_overviews_for_z;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s, ''%s'');', base_rel, base_z, overview_z, tolerance_px, has_overviews_for_z) INTO base_rel;
+    IF base_rel IS NULL THEN
+      EXIT;
+    END IF;
+    base_z := overview_z;
+    IF NOT has_overviews_for_z THEN
+      RAISE NOTICE 'Registering overview: %', base_rel;
+      PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
+    END IF;
+    SELECT array_append(overview_tables, base_rel) INTO overview_tables;
+  END LOOP;
+
+  IF overview_tables IS NOT NULL AND array_length(overview_tables, 1) > 0 THEN
+    SELECT EXISTS (
+      SELECT * FROM @extschema@.CDB_ColumnNames(reloid)  as colname WHERE colname = '_feature_count'
+    ) INTO has_counter_column;
+    IF NOT has_counter_column THEN
+      EXECUTE Format('
+        ALTER TABLE %s ADD COLUMN _feature_count integer DEFAULT 1;
+      ', reloid);
+    END IF;
+  END IF;
+
+  RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 

scripts-available/CDB_QueryStatements.sql

@@ -11,4 +11,4 @@ RETURNS SETOF TEXT AS $$
     cleaned = match[0].strip()
     if ( cleaned ):
       yield cleaned
-$$ language '@@plpythonu@@' IMMUTABLE STRICT PARALLEL SAFE;
+$$ language 'plpythonu' IMMUTABLE STRICT PARALLEL SAFE;

scripts-available/CDB_Quota.sql

@@ -20,50 +20,33 @@ RETURNS bigint AS
 $$
 DECLARE
   total_size INT8;
-  raster_available BOOLEAN;
-  raster_read_query TEXT;
 BEGIN
-  -- Postgis 3+ might not install raster
-  raster_available := EXISTS (
-        SELECT 1
-        FROM   pg_views
-        WHERE  schemaname = '@postgisschema@'
-        AND    viewname = 'raster_overviews'
-   );
-
-   IF raster_available THEN
-       raster_read_query := Format('SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews
-                        WHERE o_table_schema = %L AND o_table_catalog = current_database()', schema_name);
-   ELSE
-       raster_read_query := 'SELECT NULL::text AS o_table_name, NULL::text AS r_table_name';
-   END IF;
-   EXECUTE Format('
   WITH raster_tables AS (
-    %s
+    SELECT o_table_name, r_table_name FROM raster_overviews
+      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
   ),
   user_tables AS (
-    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(%L)
+    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(schema_name)
   ),
   table_cat AS (
     SELECT
       table_name,
       (
         EXISTS(select * from raster_tables where o_table_name = table_name)
-        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || ''[\w\d]*''
+        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || '[\w\d]*'
       ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(%L, table_name)))) table_size,
+    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(schema_name, table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1
 	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
       END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
   )
-  SELECT sum(table_size*multiplier)::int8 FROM sizes
-  ', raster_read_query, schema_name, schema_name) INTO total_size;
+  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
 
   IF total_size IS NOT NULL THEN
     RETURN total_size;

scripts-available/CDB_RegenerateTable.sql

@@ -1,153 +0,0 @@
---
--- Given a table, returns a series of queries that can be used to recreate it
--- It does not include data
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_RegenerateTable_Get_Commands(tableoid OID)
-RETURNS text[]
-AS $$
-    import subprocess
-    import re
-
-    query = "SELECT current_setting('statement_timeout') as t"
-    rv = plpy.execute(query, 1)
-    timeout_string = str(rv[0]['t'])
-    if timeout_string == "0":
-        timeout_string = "1min"
-
-    query = "SELECT current_database()::text as dname"
-    rv = plpy.execute(query, 1)
-    database_name_string = str(rv[0]['dname'])
-
-    query = """SELECT concat(quote_ident(nspname), '.', quote_ident(relname)) as quoted_name
-                        FROM pg_catalog.pg_class AS c
-                        JOIN pg_catalog.pg_namespace AS ns
-                        ON c.relnamespace = ns.oid
-                        WHERE c.oid = '%s'""" % (tableoid)
-    rv = plpy.execute(query, 1)
-    full_tablename_string = str(rv[0]['quoted_name'])
-
-    # NOTE: We always use -s so data is never dumped!
-    # That would be a security issue that we would need to deal with (and we currently do not need it)
-    process_parameters = ["pg_dump", "-s", "--lock-wait-timeout", timeout_string, "-t", full_tablename_string, database_name_string]
-
-    proc = subprocess.Popen(process_parameters, stdout=subprocess.PIPE, shell=False)
-    (out, err) = proc.communicate()
-    if (err or not out):
-        plpy.error('Could not get table properties')
-
-    line = out.decode("utf-8")
-    lines = line.rsplit(";\n", -1)
-    clean_lines = []
-    for i in range(0, len(lines)):
-        line = lines[i]
-        sublines = line.splitlines()
-        sublines = [line.rstrip() for line in sublines]
-        sublines = [line for line in sublines if line]
-        sublines = [line for line in sublines if not line.startswith('--')]
-        sublines = [line for line in sublines if not line.lower().startswith('set ')]
-        sublines = [line for line in sublines if line.lower().find('pg_catalog.set_config(') == -1]
-
-        if len(sublines):
-            clean_lines.append("".join(sublines))
-
-    return clean_lines
-$$
-LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
-
--- Returns a list of queries that can be used to regenerate the structure of a table
--- The query to create the table is not included
--- The optional parameter **ignore_cartodbfication** will remove queries related to the cartodbfication of the table
-CREATE OR REPLACE FUNCTION @extschema@.CDB_GetTableQueries(tableoid OID, ignore_cartodbfication BOOL DEFAULT false)
-RETURNS text[]
-AS
-$$
-DECLARE
-    children INTEGER;
-    queries TEXT[];
-BEGIN
-    EXECUTE FORMAT ('SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhparent =  %L', tableoid)
-            INTO children;
-    IF children > 0 THEN
-        RAISE EXCEPTION 'CDB_GetTableQueries does not support the parent of partitioned tables';
-    END IF;
-    IF NOT ignore_cartodbfication THEN
-        EXECUTE FORMAT('
-            SELECT array_agg(a)
-                FROM unnest(@extschema@.__CDB_RegenerateTable_Get_Commands(%L)) a
-                WHERE   a NOT SIMILAR TO ''CREATE TABLE%%'';', tableoid) INTO queries;
-    ELSE
-        EXECUTE FORMAT('
-            SELECT array_agg(a)
-                FROM unnest(@extschema@.__CDB_RegenerateTable_Get_Commands(%L)) a
-                WHERE   a NOT SIMILAR TO ''CREATE TABLE%%'' AND
-                        a NOT SIMILAR TO (''%%PRIMARY KEY \(cartodb_id\)%%'') AND
-                        a NOT SIMILAR TO (''%%cartodb_id_seq%%'') AND
-                        a NOT SIMILAR TO (''%%track_updates%%'') AND
-                        a NOT SIMILAR TO (''%%update_the_geom_webmercator_trigger%%'') AND
-                        a NOT SIMILAR TO (''%%test_quota%%'') AND
-                        a NOT SIMILAR TO (''%%test_quota_per_row%%'') AND
-                        a NOT SIMILAR TO (''%%gist \(the_geom\)%%'') AND
-                        a NOT SIMILAR TO (''%%gist \(the_geom_webmercator\)%%'');', tableoid) INTO queries;
-    END IF;
-    RETURN queries;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
--- Helper function to apply the result of CDB_GetTableQueries catching and discarding any exceptions
-CREATE OR REPLACE FUNCTION @extschema@.CDB_ApplyQueriesSafe(queries TEXT[])
-RETURNS void
-AS
-$$
-DECLARE
-    i INTEGER;
-BEGIN
-    IF array_length(queries, 1) > 0 THEN
-        FOR i IN 1 .. array_upper(queries, 1)
-        LOOP
-            BEGIN
-                EXECUTE queries[i];
-            EXCEPTION WHEN OTHERS THEN
-                CONTINUE;
-            END;
-        END LOOP;
-    END IF;
-END
-$$
-LANGUAGE PLPGSQL STRICT VOLATILE PARALLEL UNSAFE;
-
--- Regenerates a table
-CREATE OR REPLACE FUNCTION @extschema@.CDB_RegenerateTable(tableoid OID)
-RETURNS void
-AS
-$$
-DECLARE
-    temp_name TEXT := 'temp_' || encode(sha224(random()::text::bytea), 'hex');
-    table_name TEXT;
-    queries TEXT[] := @extschema@.__CDB_RegenerateTable_Get_Commands(tableoid);
-    i INTEGER;
-    children INTEGER;
-BEGIN
-    EXECUTE FORMAT ('SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhparent =  %L', tableoid)
-            INTO children;
-    IF children > 0 THEN
-        RAISE EXCEPTION 'CDB_RegenerateTable does not support the parent of partitioned tables';
-    END IF;
-
-    EXECUTE FORMAT('SELECT concat(quote_ident(nspname), ''.'', quote_ident(relname)) as quoted_name
-                        FROM pg_catalog.pg_class AS c
-                        JOIN pg_catalog.pg_namespace AS ns
-                        ON c.relnamespace = ns.oid
-                        WHERE c.oid = %L', tableoid) INTO table_name;
-
-    EXECUTE FORMAT('CREATE TEMPORARY TABLE %s ON COMMIT DROP AS SELECT * FROM %s', temp_name, table_name);
-    EXECUTE FORMAT('DROP TABLE %s', table_name);
-
-    FOR i IN 1 .. array_upper(queries, 1)
-    LOOP
-        EXECUTE queries[i];
-    END LOOP;
-
-    EXECUTE FORMAT('INSERT INTO %s SELECT * FROM %I', table_name, temp_name);
-END
-$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_TableIndexes.sql

@@ -16,7 +16,7 @@ AS $$
   JOIN pg_class
   ON pg_class.oid = idx.indexrelid 
   WHERE pg_indexes.tablename = '' || $1 || ''
-  AND '' || $1 || '' IN (SELECT @extschema@.CDB_UserTables())
+  AND '' || $1 || '' IN (SELECT CDB_UserTables())
   AND pg_class.relname=pg_indexes.indexname
   ;
 

scripts-available/CDB_TableMetadata.sql

@@ -43,7 +43,7 @@ BEGIN
   );
 
   WITH nv as (
-    SELECT TG_RELID as tabname, now() as t
+    SELECT TG_RELID as tabname, NOW() as t
   ), updated as (
     UPDATE @extschema@.CDB_TableMetadata x SET updated_at = nv.t
     FROM nv WHERE x.tabname = nv.tabname
@@ -55,11 +55,8 @@ BEGIN
 
   RETURN NULL;
 END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$
+LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 --
 -- Trigger invalidating varnish whenever CDB_TableMetadata
@@ -119,11 +116,8 @@ BEGIN
 
   RETURN NULL;
 END;
-$$  LANGUAGE plpgsql
-    VOLATILE
-    PARALLEL UNSAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+$$
+LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
 
 DROP TRIGGER IF EXISTS table_modified ON @extschema@.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table

scripts-available/CDB_TransformToWebmercator.sql

@@ -36,7 +36,7 @@ BEGIN
 
   -- Don't bother clipping if the geometry boundary doesn't
   -- go outside the valid extent.
-  IF @postgisschema@.geometry_within(latlon_input, valid_extent) THEN
+  IF latlon_input @ valid_extent THEN
     BEGIN
       RETURN @postgisschema@.ST_Transform(latlon_input, 3857);
     EXCEPTION WHEN OTHERS THEN

scripts-available/CDB_Username.sql

@@ -2,9 +2,5 @@
 CREATE OR REPLACE FUNCTION @extschema@.CDB_Username()
 RETURNS text
 AS $$
-  SELECT @extschema@.CDB_Conf_GetConf(concat('api_keys_', session_user))->>'username';
-$$  LANGUAGE SQL
-    STABLE
-    PARALLEL SAFE
-    SECURITY DEFINER
-    SET search_path = pg_temp;
+  SELECT @extschema@.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
+$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;

scripts-enabled/300-CDB_OAuth.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_OAuth.sql

scripts-enabled/400-CDB_FederatedServer.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_FederatedServer.sql

scripts-enabled/401_CDB_FederatedServerTables.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_FederatedServerTables.sql

scripts-enabled/402_CDB_FederatedServerListRemote.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_FederatedServerListRemote.sql

scripts-enabled/403-CDB_FederatedServerDiagnostics.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_FederatedServerDiagnostics.sql

scripts-enabled/500-CDB_RegenerateTable.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_RegenerateTable.sql

sql/test_setup.sql

@@ -1,14 +1,9 @@
-\set ECHO none
-\set QUIET on
-SET client_min_messages TO error;
 CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE SCHEMA cartodb;
-\i 'cartodb--unpackaged--@@VERSION@@.sql'
+CREATE EXTENSION plpythonu;
+CREATE EXTENSION cartodb;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
 $$ LANGUAGE 'plpgsql';
-\set QUIET off

sql/test_setup_expect

@@ -1 +0,0 @@
-\set ECHO none

test/CDB_CartodbfyTableTest.sql

@@ -134,9 +134,9 @@ DROP TABLE t;
 -- table with single non-geometrical column
 CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description;
 SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
-SELECT cartodb_id, the_geom, description FROM t;
+SELECT * FROM t;
 SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
-SELECT cartodb_id, the_geom, description FROM t;
+SELECT * FROM t;
 DROP TABLE t;
 
 -- table with existing srid-unconstrained (but type-constrained) the_geom

test/CDB_CartodbfyTableTest_expect

@@ -7,9 +7,9 @@ single non-geometrical column cartodbfied fine
 DROP TABLE
 SELECT 1
 check function idempotence cartodbfied fine
-1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
 check function idempotence cartodbfied fine
-1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
 DROP TABLE
 SELECT 1
 srid-unconstrained the_geom cartodbfied fine

test/CDB_FederatedServer.sql

@@ -1,220 +0,0 @@
--- Setup
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-SET SESSION AUTHORIZATION postgres;
-CREATE EXTENSION postgres_fdw;
-\set QUIET off
-
-\echo '## List empty servers shows nothing'
-SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## List non-existent server shows nothing'
-SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server => 'doesNotExist');
-
-\echo '## Create and list a server works'
-SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "fdw_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## Create and list a second server works'
-SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{
-    "server": {
-        "dbname": "fdw_target",
-        "host": "localhost",
-        "port": @@PGPORT@@,
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    },
-    "credentials": {
-        "username": "fdw_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## List server by name works'
-SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server => 'myRemote');
-
-
-\echo '## Re-register a second server works'
-SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{
-    "server": {
-        "dbname": "fdw_target",
-        "host": "localhost",
-        "port": @@PGPORT@@,
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    },
-    "credentials": {
-        "username": "other_remote_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## Unregister server 1 works'
-SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote'::text);
-SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## Unregistering a server that does not exist fails'
-SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server => 'doesNotExist'::text);
-
-\echo '## Unregister the second server works'
-SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote2'::text);
-SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## Create a server with NULL name fails'
-SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server => NULL::text, config => '{ "server": {}, "credentials" : {}}');
-\echo '## Create a server with NULL config fails'
-SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => NULL::jsonb);
-\echo '## Create a server with empty config fails'
-SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{}');
-\echo '## Create a server without credentials fails'
-SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
-    "server": {
-        "dbname": "fdw_target",
-        "host": "localhost",
-        "port": @@PGPORT@@,
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    }
-}'::jsonb);
-\echo '## Create a server with empty credentials works'
-SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
-    "server": {
-        "dbname": "fdw_target",
-        "host": "localhost",
-        "port": @@PGPORT@@,
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    },
-    "credentials": { }
-}'::jsonb);
-SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers();
-SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server => 'empty'::text);
-\echo '## Create a server without options fails'
-SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
-    "credentials": {
-        "username": "other_remote_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-
-\echo '## Create a server with special characters works'
-SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote" or''not'::text, config => '{
-    "server": {
-        "dbname": "fdw target",
-        "host": "localhost",
-        "port": @@PGPORT@@,
-        "extensions": "postgis",
-        "updatable": "false",
-        "use_remote_estimate": "true",
-        "fetch_size": "1000"
-    },
-    "credentials": {
-        "username": "fdw user",
-        "password": "foo barino"
-    }
-}'::jsonb);
-SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers();
-SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote" or''not'::text);
-
--- Test permissions
-\set QUIET on
-CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
-\set QUIET off
-
-SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote3'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "fdw_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-
-\c contrib_regression cdb_fs_tester
-
-\echo '## All users are able to list servers'
-SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers();
-
-\echo '## Only superadmins can create servers'
-SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "fdw_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-
-
-\c contrib_regression postgres
-
-\echo '## Granting access to a user works'
-SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT '9.55', cartodb.CDB_Federated_Server_List_Servers();
-\c contrib_regression postgres
-SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name);
-SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name);
-
-\echo '## Granting access again raises a notice'
-SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
-
-\echo '## Revoking access to a user works'
-SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
-SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
-
-\echo '## Unregistering a server with active grants works'
-SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote3'::text);
-
-
-\echo '## A user with granted access can not drop a server'
-SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "fdw_user",
-        "password": "foobarino"
-    }
-}'::jsonb);
-SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote4', db_role => 'cdb_fs_tester'::name);
-
-\c contrib_regression cdb_fs_tester
-SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text);
-
-\c contrib_regression postgres
-SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text);
-
-
--- Cleanup
-\set QUIET on
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
-DROP ROLE cdb_fs_tester;
-DROP EXTENSION postgres_fdw;
-\set QUIET off

test/CDB_FederatedServerDiagnostics.sql

@@ -1,126 +0,0 @@
--- ===================================================================
--- create FDW objects
--- ===================================================================
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-CREATE EXTENSION postgres_fdw;
-
-CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
-
--- Create database to be used as remote
-CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
-
-SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
-SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": "12345"
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
-SELECT 'C3', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback-no-port'::text, config => '{
-    "server": {
-        "host": "localhost"
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
-\c cdb_fs_tester postgres
-CREATE EXTENSION postgis;
-\c contrib_regression postgres
-\set QUIET off
-
-
--- ===================================================================
--- Test server diagnostics function(s)
--- ===================================================================
-\echo '%% It raises an error if the server does not exist'
-SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist');
-
-\echo '%% It returns a jsonb object'
-SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback'));
-
-\echo '%% It returns the server version'
-SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb
-    FROM pg_settings WHERE name = 'server_version';
-
-\echo '%% It returns the postgis version'
-SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb
-    FROM pg_extension WHERE extname = 'postgis';
-
-\echo '%% It returns null as the postgis version if it is not installed'
-\set QUIET on
-\c cdb_fs_tester postgres
-DROP EXTENSION postgis;
-\c contrib_regression postgres
-\set QUIET off
-SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb;
-
-\echo '%% It returns the remote server options'
-SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb;
-
-\echo '%% It returns network latency stats to the remote server: min <= avg <= max'
-WITH latency AS (
-   SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
-) SELECT '2.1', (latency.ms->'min')::text::float <= (latency.ms->'avg')::text::float, (latency.ms->'avg')::text::float <= (latency.ms->'max')::text::float
-FROM latency;
-
-\echo '%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)'
-WITH latency AS (
-   SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
-) SELECT '2.2', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
-FROM latency;
-
-\echo '%% Latency stats: stdev > 0'
-WITH latency AS (
-   SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
-) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0
-FROM latency;
-
-\echo '%% It raises an error if the wrong port is provided'
-SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port');
-
--- Disabled: It's not compatible with Travis since the target database (self) might be in a different port
--- \echo '%% Latency stats: can get them on default PG port 5432 when not provided'
--- WITH latency AS (
---    SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms
--- ) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
--- FROM latency;
-
-
--- ===================================================================
--- Cleanup
--- ===================================================================
-\set QUIET on
-SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text);
-SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text);
-SELECT 'D3', cartodb.CDB_Federated_Server_Unregister(server => 'loopback-no-port'::text);
--- Reconnect, using a new session in order to close FDW connections
-\connect
-DROP DATABASE cdb_fs_tester;
-
--- Drop role
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
-DROP ROLE cdb_fs_tester;
-
-DROP EXTENSION postgres_fdw;
-\set QUIET off

test/CDB_FederatedServerDiagnostics_expect

@@ -1,26 +0,0 @@
-C1|
-C2|
-C3|
-%% It raises an error if the server does not exist
-ERROR:  Server "doesNotExist" does not exist
-%% It returns a jsonb object
-1.2|jsonb
-%% It returns the server version
-1.3|t
-%% It returns the postgis version
-1.4|t
-%% It returns null as the postgis version if it is not installed
-1.5|t
-%% It returns the remote server options
-1.6|t
-%% It returns network latency stats to the remote server: min <= avg <= max
-2.1|t|t
-%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)
-2.2|t|t
-%% Latency stats: stdev > 0
-2.3|t
-%% It raises an error if the wrong port is provided
-ERROR:  could not connect to server "cdb_fs_wrong-port"
-D1|
-D2|
-D3|

test/CDB_FederatedServerListRemote.sql

@@ -1,319 +0,0 @@
--- ===================================================================
--- create FDW objects
--- ===================================================================
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-CREATE EXTENSION postgres_fdw;
-
-CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
-CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2;
-
--- Create database to be used as remote
-CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
-
-SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
-SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback2'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
--- ===================================================================
--- Setup 1
--- ===================================================================
-\c cdb_fs_tester cdb_fs_tester
-
-CREATE TYPE user_enum AS ENUM ('foo', 'bar', 'buz');
-CREATE SCHEMA "S 1";
-CREATE TABLE "S 1"."T 1" (
-	"C 1" int NOT NULL,
-	c2 int NOT NULL,
-	c3 text,
-	c4 timestamptz,
-	c5 timestamp,
-	c6 varchar(10),
-	c7 char(10),
-	c8 user_enum,
-	CONSTRAINT t1_pkey PRIMARY KEY ("C 1")
-);
-CREATE TABLE "S 1"."T 2" (
-	c1 int NOT NULL,
-	c2 text,
-	CONSTRAINT t2_pkey PRIMARY KEY (c1)
-);
-CREATE TABLE "S 1"."T 3" (
-	c1 int NOT NULL,
-	c2 int NOT NULL,
-	c3 text,
-	CONSTRAINT t3_pkey PRIMARY KEY (c1)
-);
-CREATE TABLE "S 1"."T 4" (
-	c1 int NOT NULL,
-	c2 int NOT NULL,
-	c3 text,
-	CONSTRAINT t4_pkey PRIMARY KEY (c1)
-);
-
--- Disable autovacuum for these tables to avoid unexpected effects of that
-ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
-ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
-ALTER TABLE "S 1"."T 3" SET (autovacuum_enabled = 'false');
-ALTER TABLE "S 1"."T 4" SET (autovacuum_enabled = 'false');
-
-\c contrib_regression postgres
-SET client_min_messages TO notice;
-\set VERBOSITY terse
-\set QUIET off
-
-
--- ===================================================================
--- Test listing remote schemas
--- ===================================================================
-\echo '## Test listing of remote schemas without permissions before the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote schemas (sunny day)'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
-
-\echo '## Test listing of remote schemas without permissions after the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote schemas with permissions (sunny day)'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote schemas without permissions after revoking access (rainy day)'
-SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote schemas (rainy day): Server does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist');
-
-
--- ===================================================================
--- Test listing remote tables
--- ===================================================================
-
-\echo '## Test listing of remote tables without permissions before the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote tables (sunny day)'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
-
-\echo '## Test listing of remote tables without permissions after the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote tables with permissions (sunny day)'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote tables without permissions after revoking access (rainy day)'
-SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote tables (rainy day): Server does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1');
-
-\echo '## Test listing of remote tables (rainy day): Remote schema does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist');
-
-
--- ===================================================================
--- Test listing remote columns
--- ===================================================================
-
-\echo '## Test listing of remote columns without permissions before the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote columns (sunny day)'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
-
-\echo '## Test listing of remote columns without permissions after the first instantiation (rainy day)'
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote columns with permissions (sunny day)'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote columns without permissions after revoking access (rainy day)'
-SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
-\c contrib_regression postgres
-
-\echo '## Test listing of remote columns (rainy day): Server does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1');
-
-\echo '## Test listing of remote columns (rainy day): Remote schema does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1');
-
-\echo '## Test listing of remote columns (rainy day): Remote table does not exist'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist');
-
-\echo '## Test listing of remote columns (rainy day): Remote table is NULL'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => NULL::text);
-
-
--- ===================================================================
--- Test that using a different user to list tables and dropping it
--- does not break the server: We use loopback2 as it's in a clean state
--- ===================================================================
-
-
-\echo '## Test listing of remote objects with permissions (sunny day)'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester2'::name);
-\c contrib_regression cdb_fs_tester2
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2');
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1');
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1');
-
-\c contrib_regression postgres
-\echo '## Test that dropping the granted user works fine (sunny day)'
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2;
-DROP ROLE cdb_fs_tester2;
-
-\echo '## Test listing of remote objects with other user still works (sunny day)'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2');
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1');
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1');
-
-
--- ===================================================================
--- Cleanup 1
--- ===================================================================
-\set QUIET on
-
-\c cdb_fs_tester cdb_fs_tester
-DROP TABLE "S 1". "T 1";
-DROP TABLE "S 1". "T 2";
-DROP TABLE "S 1". "T 3";
-DROP TABLE "S 1". "T 4";
-
-DROP SCHEMA "S 1";
-DROP TYPE user_enum;
-
-
--- ===================================================================
--- Setup 2: Using Postgis too
--- ===================================================================
-
-\c cdb_fs_tester postgres
-
-CREATE EXTENSION postgis;
-
-\c cdb_fs_tester cdb_fs_tester
-
-CREATE SCHEMA "S 1";
-CREATE TABLE "S 1"."T 5" (
-	geom       geometry(Geometry,4326),
-	geom_wm    geometry(GeometryZ,3857),
-	geo_nosrid geometry,
-	geog       geography
-);
-
-\c contrib_regression postgres
-SET client_min_messages TO notice;
-\set VERBOSITY terse
-\set QUIET off
-
-
--- ===================================================================
--- Test the listing functions
--- ===================================================================
-
-\echo '## Test listing of remote geometry columns (sunny day)'
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5');
-\echo '## Test listing of remote geometry columns (sunny day) - Rerun'
--- Rerun should be ok
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5');
-
-
--- ===================================================================
--- Test invalid password
--- ===================================================================
-
-\echo '## Check error message with invalid password (rainy day)'
-SELECT cartodb.CDB_Federated_Server_Register_PG(server => 'loopback_invalid'::text, config => '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "wrong password"
-    }
-}'::jsonb);
-
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid');
-
-SELECT cartodb.CDB_Federated_Server_Unregister(server => 'loopback_invalid'::text);
-
--- ===================================================================
--- Cleanup 2
--- ===================================================================
-\set QUIET on
-
-\c cdb_fs_tester cdb_fs_tester
-DROP TABLE "S 1". "T 5";
-
-DROP SCHEMA "S 1";
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-
-SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text);
-SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'loopback2'::text);
-
-DROP DATABASE cdb_fs_tester;
-
--- Drop role
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
-DROP ROLE cdb_fs_tester;
-
-DROP EXTENSION postgres_fdw;
-
-\set QUIET off

test/CDB_FederatedServerListRemote_expect

@@ -1,162 +0,0 @@
-C1|
-C2|
-## Test listing of remote schemas without permissions before the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote schemas (sunny day)
-S 1
-information_schema
-public
-## Test listing of remote schemas without permissions after the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote schemas with permissions (sunny day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-S 1
-information_schema
-public
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote schemas without permissions after revoking access (rainy day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote schemas (rainy day): Server does not exist
-ERROR:  Server "Does Not Exist" does not exist
-## Test listing of remote tables without permissions before the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote tables (sunny day)
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback"
-f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
-f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
-f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-## Test listing of remote tables without permissions after the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote tables with permissions (sunny day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback"
-f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
-f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
-f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote tables without permissions after revoking access (rainy day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote tables (rainy day): Server does not exist
-ERROR:  Server "Does Not Exist" does not exist
-## Test listing of remote tables (rainy day): Remote schema does not exist
-## Test listing of remote columns without permissions before the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote columns (sunny day)
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback"
-C 1|integer
-c2|integer
-c3|text
-c4|timestamp with time zone
-c5|timestamp without time zone
-c6|character varying
-c7|character
-c8|USER-DEFINED
-## Test listing of remote columns without permissions after the first instantiation (rainy day)
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote columns with permissions (sunny day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback"
-C 1|integer
-c2|integer
-c3|text
-c4|timestamp with time zone
-c5|timestamp without time zone
-c6|character varying
-c7|character
-c8|USER-DEFINED
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote columns without permissions after revoking access (rainy day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-You are now connected to database "contrib_regression" as user "postgres".
-## Test listing of remote columns (rainy day): Server does not exist
-ERROR:  Server "Does Not Exist" does not exist
-## Test listing of remote columns (rainy day): Remote schema does not exist
-## Test listing of remote columns (rainy day): Remote table does not exist
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback"
-## Test listing of remote columns (rainy day): Remote table is NULL
-ERROR:  Remote table name cannot be NULL
-## Test listing of remote objects with permissions (sunny day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
-S 1
-information_schema
-public
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback2"
-f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
-f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
-f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback2"
-C 1|integer
-c2|integer
-c3|text
-c4|timestamp with time zone
-c5|timestamp without time zone
-c6|character varying
-c7|character
-c8|USER-DEFINED
-You are now connected to database "contrib_regression" as user "postgres".
-## Test that dropping the granted user works fine (sunny day)
-REVOKE
-DROP ROLE
-## Test listing of remote objects with other user still works (sunny day)
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-S 1
-information_schema
-public
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback2"
-f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
-f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
-f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
-INFO:  Could not find Postgis installation in the remote "public" schema in server "loopback2"
-C 1|integer
-c2|integer
-c3|text
-c4|timestamp with time zone
-c5|timestamp without time zone
-c6|character varying
-c7|character
-c8|USER-DEFINED
-## Test listing of remote geometry columns (sunny day)
-geo_nosrid|GEOMETRY,0
-geog|Geometry,0
-geom|GEOMETRY,4326
-geom_wm|GEOMETRY,3857
-## Test listing of remote geometry columns (sunny day) - Rerun
-geo_nosrid|GEOMETRY,0
-geog|Geometry,0
-geom|GEOMETRY,4326
-geom_wm|GEOMETRY,3857
-## Check error message with invalid password (rainy day)
-
-ERROR:  could not connect to server "cdb_fs_loopback_invalid"
-
-D1|
-D2|

test/CDB_FederatedServerTables.sql

@@ -1,405 +0,0 @@
--- ===================================================================
--- create FDW objects
--- ===================================================================
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-
-CREATE EXTENSION postgres_fdw;
-
--- We create a username following the same steps as organization members
-CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
-
-CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2;
-
--- Create database to be used as remote
-CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
-
-SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{
-    "server": {
-        "host": "localhost",
-        "port": @@PGPORT@@
-    },
-    "credentials": {
-        "username": "cdb_fs_tester",
-        "password": "cdb_fs_passwd"
-    }
-}'::jsonb);
-
-
--- ===================================================================
--- create objects used through FDW loopback server
--- ===================================================================
-
-\c cdb_fs_tester postgres
-
-CREATE EXTENSION postgis;
-
-\c cdb_fs_tester cdb_fs_tester
-
-CREATE SCHEMA remote_schema;
-CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326));
-
-INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry);
-INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry);
-
-CREATE TABLE remote_schema.remote_geom2(cartodb_id bigint, another_field text, the_geom geometry(Geometry,4326), the_geom_webmercator geometry(Geometry,3857));
-
-INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)');
-
-CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text);
-INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes');
-
-CREATE TABLE remote_schema.remote_geom3(id bigint, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857));
-
--- ===================================================================
--- Test the listing functions
--- ===================================================================
-
-\c contrib_regression postgres
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-\echo '## Registering an existing table works'
-SELECT 'R1', cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback.remote_geom;
-
-Select * FROM CDB_Federated_Server_List_Remote_Tables(
-    server => 'loopback',
-    remote_schema => 'remote_schema'
-);
-
-\echo '## Registering another existing table works'
-SELECT 'R2', cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom2',
-    id_column => 'cartodb_id',
-    geom_column => 'the_geom',
-    webmercator_column => 'the_geom_webmercator',
-    local_name => 'myFullTable'
-    );
-
-SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback."myFullTable";
-Select * FROM CDB_Federated_Server_List_Remote_Tables(
-    server => 'loopback',
-    remote_schema => 'remote_schema'
-);
-
-\echo '## Re-registering a table works'
-SELECT 'R3', cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom2',
-    id_column => 'cartodb_id',
-    -- Switch geom and geom_column on purpose to force ST_Transform to be used
-    geom_column => 'the_geom_webmercator',
-    webmercator_column => 'the_geom',
-    local_name => 'different_name'
-    );
-
--- The old view should dissapear
-SELECT 'S3_old', cartodb_id, another_field FROM cdb_fs_loopback."myFullTable";
--- And the new appear
-SELECT 'S3_new', cartodb_id, another_field FROM cdb_fs_loopback.different_name;
-
-\echo '## Unregistering works'
--- Deregistering the first table
-SELECT 'U1', cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom'
-    );
--- Selecting from the created view should fail now
-SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom;
-
-Select * FROM CDB_Federated_Server_List_Remote_Tables(
-    server => 'loopback',
-    remote_schema => 'remote_schema'
-);
-
--- ===================================================================
--- Test input
--- ===================================================================
-
-\echo '## Registering a table: Invalid server fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'Does not exist',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: NULL server fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => NULL::text,
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: Invalid schema fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'Does not exist',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: NULL schema fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => NULL::text,
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: Invalid table fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'Does not exist',
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: NULL table fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => NULL::text,
-    id_column => 'id',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: Invalid id fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'Does not exist',
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: NULL id fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  NULL::text,
-    geom_column => 'geom'
-    );
-
-\echo '## Registering a table: Invalid geom_column fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'Does not exists'
-    );
-
-\echo '## Registering a table: NULL geom_column is OK: Reuses geom_mercator'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => NULL::text,
-    webmercator_column => 'geom'
-    );
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom';
-
-SELECT cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom'
-    );
-
-\echo '## Registering a table: Invalid webmercator_column fails'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column => 'id',
-    geom_column => 'geom',
-    webmercator_column => 'Does not exists'
-    );
-
-\echo '## Registering a table: NULL webmercator_column is OK: Reuses geom'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => 'geom',
-    webmercator_column => NULL::text
-    );
-SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom';
-SELECT cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom'
-    );
-
-\echo '##Registering a table with extra columns show the correct information'
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom3',
-    id_column =>  'id',
-    geom_column => 'geom',
-    webmercator_column => 'geom_mercator'
-    );
-Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom3';
-SELECT cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom3'
-    );
-
--- ===================================================================
--- Test conflicts
--- ===================================================================
-
-\echo '## Target conflict is handled nicely: Table'
-CREATE TABLE cdb_fs_loopback.localtable (a integer);
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => 'geom',
-    local_name => 'localtable');
-
-\echo '## Target conflict is handled nicely: View'
-CREATE VIEW cdb_fs_loopback.localtable2 AS Select * from cdb_fs_loopback.localtable;
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => 'geom',
-    local_name => 'localtable2');
-
-DROP VIEW cdb_fs_loopback.localtable2;
-DROP TABLE cdb_fs_loopback.localtable;
-
--- ===================================================================
--- Test permissions
--- ===================================================================
-
-
-\echo '## Registering tables does not work without permissions'
-\c contrib_regression cdb_fs_tester
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => 'geom',
-    local_name => 'localtable');
-
-\echo '## Normal users can not write in the import schema'
-CREATE TABLE cdb_fs_loopback.localtable (a integer);
-
-\echo '## Listing remote tables does not work without permissions'
-Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
-
-\echo '## Registering tables works with granted permissions'
-\c contrib_regression postgres
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name);
-\c contrib_regression cdb_fs_tester
-SELECT cartodb.CDB_Federated_Table_Register(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom',
-    id_column =>  'id',
-    geom_column => 'geom',
-    local_name => 'localtable');
-
-\echo '## Listing remote tables works with granted permissions'
-Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
-
-\echo '## Selecting from a registered table with granted permissions works'
-Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable;
-
-\echo '## Selecting from a registered table without permissions does not work'
-\c contrib_regression cdb_fs_tester2
-CREATE OR REPLACE FUNCTION catch_permission_error(query text)
-RETURNS bool
-AS $$
-BEGIN
-    EXECUTE query;
-    RETURN FALSE;
-EXCEPTION
-    WHEN insufficient_privilege THEN
-        RETURN TRUE;
-    WHEN OTHERS THEN
-        RAISE WARNING 'Exception %', sqlstate;
-        RETURN FALSE;
-END
-$$ LANGUAGE 'plpgsql';
-Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable$$);
-DROP FUNCTION catch_permission_error(text);
-
-\echo '## Deleting a registered table without permissions does not work'
-SELECT cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom'
-    );
-
-\echo '## Only the owner can grant permissions over the server'
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name);
-
-\echo '## Everything works for a different user when granted permissions'
-\c contrib_regression postgres
-SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name);
-\c contrib_regression cdb_fs_tester2
-Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
-Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable;
-
-\echo '## A different user can unregister a table'
-SELECT cartodb.CDB_Federated_Table_Unregister(
-    server => 'loopback',
-    remote_schema => 'remote_schema',
-    remote_table => 'remote_geom'
-    );
-Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
-
-\echo '## Only the owner can revoke permissions over the server'
-SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name);
-
--- ===================================================================
--- Cleanup
--- ===================================================================
-
-\set QUIET on
-\c contrib_regression postgres
-SET client_min_messages TO error;
-\set VERBOSITY terse
-
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2;
-DROP ROLE cdb_fs_tester2;
-
-SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text);
-DROP DATABASE cdb_fs_tester;
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
-DROP ROLE cdb_fs_tester;
-DROP EXTENSION postgres_fdw;
-\set QUIET off

test/CDB_FederatedServerTables_expect

@@ -1,116 +0,0 @@
-C1|
-## Registering an existing table works
-R1|
-S1|1|POINT(1 1)|patata
-S1|2|POINT(2 2)|patata2
-t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-## Registering another existing table works
-R2|
-S2|3|POINT(3 3)|patata
-t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-t|remote_geom2|cdb_fs_loopback."myFullTable"|cartodb_id|the_geom|the_geom_webmercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-## Re-registering a table works
-R3|
-ERROR:  relation "cdb_fs_loopback.myFullTable" does not exist at character 49
-S3_new|3|patata
-## Unregistering works
-U1|
-ERROR:  relation "remote_geom" does not exist at character 71
-f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-## Registering a table: Invalid server fails
-ERROR:  Server "Does not exist" does not exist
-## Registering a table: NULL server fails
-ERROR:  Server name cannot be NULL
-## Registering a table: Invalid schema fails
-ERROR:  Could not import schema "Does not exist" of server "loopback": schema "Does not exist" is not present on foreign server "cdb_fs_loopback"
-## Registering a table: NULL schema fails
-ERROR:  Schema name cannot be NULL
-## Registering a table: Invalid table fails
-ERROR:  Could not import table "remote_schema.Does not exist" of server "loopback"
-## Registering a table: NULL table fails
-ERROR:  Remote table name cannot be NULL
-## Registering a table: Invalid id fails
-ERROR:  non integer id_column "Does not exist"
-## Registering a table: NULL id fails
-ERROR:  non integer id_column "<NULL>"
-## Registering a table: Invalid geom_column fails
-ERROR:  non geometry column "Does not exists"
-## Registering a table: NULL geom_column is OK: Reuses geom_mercator
-
-t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-
-## Registering a table: Invalid webmercator_column fails
-ERROR:  non geometry column "Does not exists"
-## Registering a table: NULL webmercator_column is OK: Reuses geom
-
-t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-
-##Registering a table with extra columns show the correct information
-
-t|remote_geom3|cdb_fs_loopback.remote_geom3|id|geom|geom_mercator|[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-
-## Target conflict is handled nicely: Table
-CREATE TABLE
-ERROR:  Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists
-## Target conflict is handled nicely: View
-CREATE VIEW
-ERROR:  Could not import table "remote_geom" as "cdb_fs_loopback.localtable2" already exists
-DROP VIEW
-DROP TABLE
-## Registering tables does not work without permissions
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to access the server "loopback"
-## Normal users can not write in the import schema
-ERROR:  permission denied for schema cdb_fs_loopback at character 14
-## Listing remote tables does not work without permissions
-ERROR:  Not enough permissions to access the server "loopback"
-## Registering tables works with granted permissions
-You are now connected to database "contrib_regression" as user "postgres".
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-
-## Listing remote tables works with granted permissions
-t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-## Selecting from a registered table with granted permissions works
-1|POINT(1 1)
-2|POINT(2 2)
-## Selecting from a registered table without permissions does not work
-You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
-CREATE FUNCTION
-t
-DROP FUNCTION
-## Deleting a registered table without permissions does not work
-ERROR:  Not enough permissions to access the server "loopback"
-## Only the owner can grant permissions over the server
-ERROR:  You do not have rights to grant access on "loopback"
-## Everything works for a different user when granted permissions
-You are now connected to database "contrib_regression" as user "postgres".
-
-You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
-t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-1|POINT(1 1)
-2|POINT(2 2)
-## A different user can unregister a table
-NOTICE:  drop cascades to view cdb_fs_loopback.localtable
-
-f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
-t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
-f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
-f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
-## Only the owner can revoke permissions over the server
-ERROR:  You do not have rights to revoke access on "loopback"
-D1|

test/CDB_FederatedServer_expect

@@ -1,69 +0,0 @@
-## List empty servers shows nothing
-## List non-existent server shows nothing
-## Create and list a server works
-1.3|
-1.4|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
-## Create and list a second server works
-2.1|
-2.2|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
-2.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,fdw_user)
-## List server by name works
-2.3|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
-## Re-register a second server works
-3.1|
-3.2|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
-3.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,other_remote_user)
-## Unregister server 1 works
-4.1|
-4.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,other_remote_user)
-## Unregistering a server that does not exist fails
-ERROR:  Server "doesNotExist" does not exist
-## Unregister the second server works
-6.1|
-## Create a server with NULL name fails
-ERROR:  Server name cannot be NULL
-## Create a server with NULL config fails
-7.01|
-## Create a server with empty config fails
-ERROR:  Server information is mandatory
-## Create a server without credentials fails
-ERROR:  Credentials are mandatory
-## Create a server with empty credentials works
-7.3|
-7.4|(empty,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,)
-7.5|
-## Create a server without options fails
-ERROR:  Server information is mandatory
-## Create a server with special characters works
-8.1|
-8.2|("myRemote"" or'not",postgres_fdw,localhost,@@PGPORT@@,"fdw target",read-only,"fdw user")
-8.3|
-9.1|
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-## All users are able to list servers
-9.2|(myRemote3,postgres_fdw,localhost,@@PGPORT@@,,read-only,)
-## Only superadmins can create servers
-ERROR:  Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw
-You are now connected to database "contrib_regression" as user "postgres".
-## Granting access to a user works
-9.5|
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-9.55|(myRemote3,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
-You are now connected to database "contrib_regression" as user "postgres".
-ERROR:  Server "does not exist" does not exist
-ERROR:  Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist
-## Granting access again raises a notice
-NOTICE:  role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368"
-9.8|
-## Revoking access to a user works
-9.9|
-9.10|
-## Unregistering a server with active grants works
-9.11|
-## A user with granted access can not drop a server
-10.1|
-10.2|
-You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-ERROR:  Not enough permissions to drop the server "myRemote4"
-You are now connected to database "contrib_regression" as user "postgres".
-10.4|

test/CDB_GhostTables.sql

@@ -1,56 +1,6 @@
 -- Create user and enable Ghost tables trigger
 \set QUIET on
 SET client_min_messages TO error;
-
--- Recreate the function without extra error messages as it changes depending on the python-redis version
-CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text)
-RETURNS void
-AS $$
-  if not username:
-    return
-
-  if 'json' not in GD:
-    import json
-    GD['json'] = json
-  else:
-    json = GD['json']
-
-  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
-  if not tis_config:
-    plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
-    return
-
-  tis_config_dict = json.loads(tis_config)
-  tis_host = tis_config_dict.get('host')
-  tis_port = tis_config_dict.get('port')
-  tis_timeout = tis_config_dict.get('timeout', 5)
-  tis_retry = tis_config_dict.get('retry', 5)
-
-  client = GD.get('invalidation', None)
-
-  while True:
-
-    if not client:
-        try:
-          import redis
-          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
-          GD['invalidation'] = client
-        except Exception as err:
-          # NOTE: no retries on connection error
-          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables')
-          break
-
-    try:
-      client.execute_command('DBSCH', db_name, username, event_name)
-      break
-    except Exception as err:
-      client = GD['invalidation'] = None # force reconnect
-      if not tis_retry:
-        plpy.warning('Error calling Invalidation Service to link Ghost Tables')
-        break
-      tis_retry -= 1 # try reconnecting
-$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
-
 SELECT CDB_EnableGhostTablesTrigger();
 CREATE ROLE "fulano" LOGIN;
 GRANT ALL ON SCHEMA cartodb TO "fulano";

test/CDB_GhostTables_expect

@@ -1,19 +1,19 @@
 
 
 WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
-INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 
-WARNING:  Error calling Invalidation Service to link Ghost Tables
-INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 BEGIN
 cdb_ddl_execution
 0
 CREATE TABLE
 1
-WARNING:  Error calling Invalidation Service to link Ghost Tables
-INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
 COMMIT
 
 

test/CDB_GreatCircle.sql

@@ -1,2 +1,2 @@
-select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)), 3);
-select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000), 3);
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)));
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000));

test/CDB_GreatCircle_expect

@@ -1,2 +1,2 @@
-LINESTRING(4.259 55.858,5.533 56.001,6.817 56.13,8.109 56.246,9.408 56.349,10.713 56.437,12.025 56.513,13.341 56.574,14.661 56.621,15.983 56.654,17.308 56.674,18.633 56.679,19.958 56.67,21.282 56.647,22.604 56.61,23.923 56.559,25.238 56.494,26.548 56.415,27.852 56.323,29.149 56.217,30.439 56.097,31.72 55.964,32.992 55.818,34.254 55.659,35.505 55.487,36.745 55.302,37.974 55.105,39.189 54.896,40.392 54.674,41.582 54.441,42.758 54.197,43.919 53.941,45.066 53.674,46.199 53.396,47.316 53.107,48.418 52.809,49.505 52.5,50.577 52.181,51.633 51.853,52.673 51.515,53.698 51.169,54.708 50.813,55.702 50.449,56.681 50.077,57.644 49.697,58.593 49.308,59.526 48.912,60.445 48.509,61.349 48.098,62.238 47.681,63.114 47.257,63.975 46.826,64.822 46.388,65.656 45.945,66.477 45.496,67.284 45.041,68.079 44.58,68.861 44.114,69.631 43.642,70.389 43.166,71.135 42.684,71.869 42.198,72.592 41.708,73.304 41.212,74.006 40.713)
-LINESTRING(4.259 55.858,4.895 55.931,5.533 56.001,6.174 56.067,6.817 56.13,7.462 56.19,8.109 56.246,8.757 56.299,9.408 56.349,10.06 56.395,10.713 56.437,11.368 56.477,12.025 56.513,12.682 56.545,13.341 56.574,14 56.599,14.661 56.621,15.322 56.639,15.983 56.654,16.645 56.666,17.308 56.674,17.97 56.678,18.633 56.679,19.296 56.676,19.958 56.67,20.62 56.66,21.282 56.647,21.943 56.63,22.604 56.61,23.264 56.586,23.923 56.559,24.581 56.528,25.238 56.494,25.893 56.456,26.548 56.415,27.2 56.371,27.852 56.323,28.501 56.271,29.149 56.217,29.795 56.159,30.439 56.097,31.08 56.032,31.72 55.964,32.357 55.893,32.992 55.818,33.624 55.74,34.254 55.659,34.881 55.574,35.505 55.487,36.127 55.396,36.745 55.302,37.361 55.205,37.974 55.105,38.583 55.002,39.189 54.896,39.792 54.787,40.392 54.674,40.989 54.559,41.582 54.441,42.171 54.32,42.758 54.197,43.34 54.07,43.919 53.941,44.494 53.809,45.066 53.674,45.634 53.536,46.199 53.396,46.759 53.253,47.316 53.107,47.869 52.959,48.418 52.809,48.964 52.655,49.505 52.5,50.043 52.342,50.577 52.181,51.107 52.018,51.633 51.853,52.155 51.685,52.673 51.515,53.188 51.343,53.698 51.169,54.205 50.992,54.708 50.813,55.207 50.632,55.702 50.449,56.193 50.264,56.681 50.077,57.164 49.888,57.644 49.697,58.12 49.503,58.593 49.308,59.061 49.111,59.526 48.912,59.987 48.712,60.445 48.509,60.899 48.305,61.349 48.098,61.795 47.89,62.238 47.681,62.678 47.47,63.114 47.257,63.546 47.042,63.975 46.826,64.4 46.608,64.822 46.388,65.241 46.168,65.656 45.945,66.068 45.721,66.477 45.496,66.882 45.269,67.284 45.041,67.683 44.811,68.079 44.58,68.472 44.348,68.861 44.114,69.248 43.879,69.631 43.642,70.011 43.405,70.389 43.166,70.763 42.926,71.135 42.684,71.503 42.442,71.869 42.198,72.232 41.953,72.592 41.708,72.95 41.46,73.304 41.212,73.657 40.963,74.006 40.713)
+LINESTRING(4.259 55.858,5.53349240387128 56.0006659105918,6.81698919498694 56.130094578525,8.10870381314147 56.2461317260662,9.40781156033233 56.3486370295466,10.7134527044527 56.4374849223869,12.0247359780093 56.5125653297878,13.3407424424749 56.573784325367,14.660529681225 56.6210647008095,15.9831362768927 56.654346440595,17.307586522649 56.67358709506,18.6328953115992 56.6787620464102,19.9580731443722 56.6698646638042,21.282131192215 56.6469063452276,22.6040863516019 56.6099164455407,23.922966226566 56.5589420917603,25.2378139766594 56.4940478882858,26.5476929715805 56.4153155163602,27.8516911979552 56.3228432335229,29.1489253693643 56.2167452801302,30.4385446972665 56.0971512011646,31.7197342877491 55.9642050924945,32.9917181368037 55.8180647814723,34.2537617048216 55.6589009522625,35.5051740589896 55.4868962265697,36.7453095800251 55.3022442104976,37.9735692370026 55.1051485181267,39.1894014407465 54.8958217820713,40.3923024922398 54.6744846607816,41.5818166476476 54.4413648517294,42.757535825811 54.1966961188706,43.919098987406 53.9407173419567,45.0661912174019 53.673671594382,46.198542544017 53.3958052553427,47.3159265281308 53.1073671611612,48.4181586571351 52.8086077997244,49.5050945765883 52.499778551104,50.5766281918714 52.1811309766006,51.6326896704254 51.8529161576737,52.673243373185 51.5153840855177,53.6982857415906 51.1687831014009,54.7078431641625 50.8133593873253,55.7019698441171 50.4493565060761,56.6807456869812 50.0770149893128,57.6442742246566 49.6965719720156,58.5926805899637 49.3082608713202,59.5261095533829 48.9123111075629,60.4447236315382 48.5089478652008,61.3487012749643 48.0983918911668,62.2382351408597 47.6808593281578,63.1135304548766 47.2565615803358,63.9748034645285 46.8257052089336,64.822279985501 46.3884918552974,65.6561940410346 45.9451181889661,66.476786593589 45.4957758784676,67.284304367196 45.0406515826125,68.0789987582454 44.5799269601738,68.8611248319107 44.1137786959568,69.6309404010034 43.6423785413868,70.388705183725 43.1658933678633,71.1346800365587 42.6844852312539,71.8691262583921 42.1983114460249,72.5923049618788 41.7075246676227,73.3044765080245 41.2122729818388,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.89507305967085 55.930977446384,5.53349240387128 56.0006659105918,6.17416348361598 56.0670448594645,6.81698919498694 56.130094578525,7.46186995983655 56.1897961993418,8.10870381314147 56.2461317260662,8.75738649688733 56.2990840610623,9.40781156033233 56.3486370295466,10.0598704664666 56.3947754031591,10.7134527044527 56.4374849223869,11.3684459078018 56.4767523177655,12.0247359780093 56.5125653297878,12.6822072133468 56.5449127274491,13.3407424424749 56.573784325367,14.0002231625192 56.5991709994144,14.660529681225 56.6210647008095,15.3215412627822 56.6394584686143,15.9831362768927 56.654346440595,16.6451923506331 56.6657238624055,17.307586522649 56.67358709506,17.9701953992046 56.677933620668,18.6328953115992 56.6787620464102,19.2955624744544 56.6760721067401,19.9580731443722 56.6698646638042,20.6203037784591 56.6601417060788,21.282131192215 56.6469063452276,21.943432716288 56.6301628111935,22.6040863516019 56.6099164455407,23.2639709223762 56.5861736930735,23.922966226566 56.5589420917603,24.5809531832687 56.5282302610022,25.2378139766594 56.4940478882858,25.8934321960358 56.4564057142701,26.5476929715805 56.4153155163602,27.2004831054654 56.3707900908252,27.8516911979552 56.3228432335229,28.5012077681911 56.2714897192993,29.1489253693643 56.2167452801302,29.7947386980206 56.1586265820819,30.4385446972665 56.0971512011646,31.0802426536785 56.0323375981587,31.7197342877491 55.9642050924945,32.3569238377352 55.8927738352675,32.9917181368037 55.8180647814723,33.6240266834038 55.74009966154,34.2537617048216 55.6589009522625,34.8808382139074 55.5744918471876,35.5051740589896 55.4868962265697,36.1266899670207 55.3961386269571,36.7453095800251 55.3022442104976,37.3609594849451 55.2052387340427,37.9735692370026 55.1051485181267,38.5830713767178 55.0020004158976,39.1894014407465 54.8958217820713,39.7924979667135 54.7866404419798,40.3923024922398 54.6744846607816,40.9887595483734 54.5593831128969,41.5818166476476 54.4413648517294,42.1714242670021 54.3204592797319,42.757535825811 54.1966961188706,43.3401076592708 54.0701053815371,43.919098987406 53.9407173419567,44.4944718799548 53.8085625081347,45.0661912174019 53.673671594382,45.6342246484243 53.5360754944551,46.198542544017 53.3958052553427,46.7591179485663 53.2528920517295,47.3159265281308 53.1073671611612,47.8689465161932 52.9592619399335,48.4181586571351 52.8086077997244,48.9635461476859 52.6554361849853,49.5050945765883 52.499778551104,50.0427918627159 52.3416663433486,50.5766281918714 52.1811309766006,51.1065959524853 52.0182038158815,51.6326896704254 51.8529161576737,52.1549059431199 51.6852992120372,52.673243373185 51.5153840855177,53.1877025017413 51.3432017648431,53.6982857415906 51.1687831014009,54.2049973104167 50.9921587964881,54.7078431641625 50.8133593873253,55.2068309307272 50.6324152338211,55.7019698441171 50.4493565060761,56.1932706791714 50.2642131726125,56.6807456869812 50.0770149893128,57.1644085311015 49.8877914890534,57.6442742246566 49.6965719720156,58.1203590684218 49.5033854966561,58.5926805899637 49.3082608713202,59.0612574839055 49.1112266464775,59.5261095533829 48.9123111075629,59.9872576527434 48.7115422684016,60.4447236315382 48.5089478652008,60.8985302798459 48.3045553510865,61.3487012749643 48.0983918911668,61.7952611294973 47.8904843581013,62.2382351408597 47.6808593281578,62.6776493422177 47.4695430777354,63.1135304548766 47.2565615803358,63.5459058421237 47.0419405039633,63.9748034645285 46.8257052089336,64.4002518367009 46.6078807460742,64.822279985501 46.3884918552974,65.2409174096934 46.1675629645276,65.6561940410346 45.9451181889661,66.0681402067793 45.7211813306754,66.476786593589 45.4957758784676,66.8821642128236 45.2689250080781,67.284304367196 45.0406515826125,67.6832386187654 44.8109781532476,68.0789987582454 44.5799269601738,68.4716167756018 44.3475199337644,68.8611248319107 44.1137786959568,69.2475552324516 43.878724561833,69.6309404010034 43.6423785413868,70.0113128553159 43.4047613414637,70.388705183725 43.1658933678633,70.7631500228809 42.925794727592,71.1346800365587 42.6844852312539,71.5033278955199 42.4419843955718,71.8691262583921 42.1983114460249,72.2321077535378 41.953485319597,72.5923049618788 41.7075246676227,72.9497504006463 41.4604478587259,73.3044765080245 41.2122729818388,73.6565156286596 40.963017849297,74.0059 40.7127)

test/CDB_OAuth.sql

@@ -1,223 +0,0 @@
--- Create user and enable OAuth event trigger
-\set QUIET on
-SET client_min_messages TO error;
-
--- The permission error changed between pre PG11 and post 11 (before everything was "relation", now it's "view", "table" and so on
-CREATE OR REPLACE FUNCTION catch_permission_error(query text)
-    RETURNS bool
-AS $$
-BEGIN
-    EXECUTE query;
-    RETURN FALSE;
-EXCEPTION
-    WHEN insufficient_privilege THEN
-        RETURN TRUE;
-    WHEN OTHERS THEN
-        RAISE WARNING 'Exception %', sqlstate;
-        RETURN FALSE;
-END
-$$ LANGUAGE 'plpgsql';
-
-DROP ROLE IF EXISTS "creator_role";
-CREATE ROLE "creator_role" LOGIN;
-DROP ROLE IF EXISTS "ownership_role";
-CREATE ROLE "ownership_role" LOGIN;
-GRANT ALL ON SCHEMA cartodb TO "creator_role";
-SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[]}');
-SET SESSION AUTHORIZATION "creator_role";
-SET client_min_messages TO notice;
-\set QUIET off
-
--- First part without event trigger
-
-CREATE TABLE test(id INT);
-INSERT INTO test VALUES(1);
-CREATE TABLE test_tablesas AS SELECT * FROM test;
-CREATE VIEW test_view AS SELECT * FROM test;
-CREATE MATERIALIZED VIEW test_mview AS SELECT * FROM test;
-SELECT * INTO test_selectinto FROM test;
-CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-
-SELECT * FROM test;
-SELECT * FROM test_tablesas;
-SELECT * FROM test_view;
-SELECT * FROM test_mview;
-SELECT * FROM test_selectinto;
-SELECT test_function();
--- Postgres grants default execute privilege on functions to PUBLIC. So in order to check the different permissions
--- between creator and owner roles is not enough with performing a selection, we need to DROP the table (which only the owner can do)
-DROP FUNCTION test_function();
-
-\set QUIET on
-CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-SET SESSION AUTHORIZATION "ownership_role";
-\set QUIET off
-
-SELECT 'denied_table', catch_permission_error($$SELECT * FROM test;$$);
-SELECT 'denied_tableas', catch_permission_error($$SELECT * FROM test_tablesas;$$);
-SELECT 'denied_view', catch_permission_error($$SELECT * FROM test_view;$$);
-SELECT 'denied_mview', catch_permission_error($$SELECT * FROM test_mview;$$);
-SELECT 'denied_selectinto', catch_permission_error($$SELECT * FROM test_selectinto;$$);
-SELECT test_function();
-SELECT 'denied_function', catch_permission_error($$DROP FUNCTION test_function();$$);
-
-\set QUIET on
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-DROP TABLE test_tablesas;
-DROP VIEW test_view;
-DROP MATERIALIZED VIEW test_mview;
-DROP TABLE test_selectinto;
-DROP TABLE test;
-DROP FUNCTION test_function();
-
--- Second part with event trigger but without ownership_role_name in cdb_conf
-
-\set QUIET on
-SET SESSION AUTHORIZATION postgres;
-SELECT CDB_EnableOAuthReassignTablesTrigger();
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-CREATE TABLE test2(id INT);
-INSERT INTO test2 VALUES(1);
-CREATE TABLE test2_tablesas AS SELECT * FROM test2;
-CREATE VIEW test2_view AS SELECT * FROM test2;
-CREATE MATERIALIZED VIEW test2_mview AS SELECT * FROM test2;
-SELECT * INTO test2_selectinto FROM test2;
-CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-
-SELECT * FROM test2;
-SELECT * FROM test2_tablesas;
-SELECT * FROM test2_view;
-SELECT * FROM test2_mview;
-SELECT * FROM test2_selectinto;
-SELECT test2_function();
-DROP FUNCTION test2_function();
-
-\set QUIET on
-CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-SET SESSION AUTHORIZATION "ownership_role";
-\set QUIET off
-
-SELECT 'denied_table2', catch_permission_error($$SELECT * FROM test2;$$);
-SELECT 'denied_tableas2', catch_permission_error($$SELECT * FROM test2_tablesas;$$);
-SELECT 'denied_view2', catch_permission_error($$SELECT * FROM test2_view;$$);
-SELECT 'denied_mview2', catch_permission_error($$SELECT * FROM test2_mview;$$);
-SELECT 'denied_selectinto2', catch_permission_error($$SELECT * FROM test2_selectinto;$$);
-SELECT test2_function();
-SELECT 'denied_function2', catch_permission_error($$DROP FUNCTION test2_function();$$);
-
-\set QUIET on
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-DROP TABLE test2_tablesas;
-DROP VIEW test2_view;
-DROP MATERIALIZED VIEW test2_mview;
-DROP TABLE test2_selectinto;
-DROP TABLE test2;
-DROP FUNCTION test2_function();
-
--- Third part with event trigger but with empty ownership_role_name in cdb_conf
-
-\set QUIET on
-SET SESSION AUTHORIZATION postgres;
-SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": ""}');
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-CREATE TABLE test3(id INT);
-INSERT INTO test3 VALUES(1);
-CREATE TABLE test3_tablesas AS SELECT * FROM test3;
-CREATE VIEW test3_view AS SELECT * FROM test3;
-CREATE MATERIALIZED VIEW test3_mview AS SELECT * FROM test3;
-SELECT * INTO test3_selectinto FROM test3;
-CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-
-SELECT * FROM test3;
-SELECT * FROM test3_tablesas;
-SELECT * FROM test3_view;
-SELECT * FROM test3_mview;
-SELECT * FROM test3_selectinto;
-SELECT test3_function();
-DROP FUNCTION test3_function();
-
-\set QUIET on
-CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-SET SESSION AUTHORIZATION "ownership_role";
-\set QUIET off
-
-SELECT 'denied_table3', catch_permission_error($$SELECT * FROM test3;$$);
-SELECT 'denied_tableas3', catch_permission_error($$SELECT * FROM test3_tablesas;$$);
-SELECT 'denied_view3', catch_permission_error($$SELECT * FROM test3_view;$$);
-SELECT 'denied_mview3', catch_permission_error($$SELECT * FROM test3_mview;$$);
-SELECT 'denied_selectinto3', catch_permission_error($$SELECT * FROM test3_selectinto;$$);
-SELECT test3_function();
-SELECT 'denied_function3', catch_permission_error($$DROP FUNCTION test3_function();$$);
-
-\set QUIET on
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-DROP TABLE test3_tablesas;
-DROP VIEW test3_view;
-DROP MATERIALIZED VIEW test3_mview;
-DROP TABLE test3_selectinto;
-DROP TABLE test3;
-DROP FUNCTION test3_function();
-
--- Fourth part with the event trigger active and configured
-
-\set QUIET on
-SET SESSION AUTHORIZATION postgres;
-SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": "ownership_role"}');
-SET SESSION AUTHORIZATION "creator_role";
-\set QUIET off
-
-CREATE TABLE test4(id INT);
-INSERT INTO test4 VALUES(1);
-CREATE TABLE test4_tablesas AS SELECT * FROM test4;
-CREATE VIEW test4_view AS SELECT * FROM test4;
-CREATE MATERIALIZED VIEW test4_mview AS SELECT * FROM test4;
-SELECT * INTO test4_selectinto FROM test4;
-CREATE FUNCTION test4_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
-
-SELECT * FROM test4;
-SELECT * FROM test4_tablesas;
-SELECT * FROM test4_view;
-SELECT * FROM test4_mview;
-SELECT * FROM test4_selectinto;
-SELECT test4_function();
-SELECT 'denied_function4', catch_permission_error($$DROP FUNCTION test4_function();$$);
-
-\set QUIET on
-SET SESSION AUTHORIZATION "ownership_role";
-\set QUIET off
-
-SELECT * FROM test4;
-SELECT * FROM test4_tablesas;
-SELECT * FROM test4_view;
-SELECT * FROM test4_mview;
-SELECT * FROM test4_selectinto;
-SELECT test4_function();
-
--- Ownership role drops the tables
-DROP TABLE test4_tablesas;
-DROP VIEW test4_view;
-DROP MATERIALIZED VIEW test4_mview;
-DROP TABLE test4_selectinto;
-DROP TABLE test4;
-DROP FUNCTION test4_function();
-
--- Cleanup
-\set QUIET on
-SET SESSION AUTHORIZATION postgres;
-SELECT CDB_DisableOAuthReassignTablesTrigger();
-DROP ROLE "ownership_role";
-REVOKE ALL ON SCHEMA cartodb FROM "creator_role";
-DROP ROLE "creator_role";
-DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role';
-DROP FUNCTION catch_permission_error(text);
-\set QUIET off

test/CDB_OAuth_expect

@@ -1,113 +0,0 @@
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-CREATE FUNCTION
-1
-1
-1
-1
-1
-1
-DROP FUNCTION
-denied_table|t
-denied_tableas|t
-denied_view|t
-denied_mview|t
-denied_selectinto|t
-1
-denied_function|t
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-DROP FUNCTION
-NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-CREATE FUNCTION
-1
-1
-1
-1
-1
-1
-DROP FUNCTION
-denied_table2|t
-denied_tableas2|t
-denied_view2|t
-denied_mview2|t
-denied_selectinto2|t
-1
-denied_function2|t
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-DROP FUNCTION
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-CREATE FUNCTION
-1
-1
-1
-1
-1
-1
-DROP FUNCTION
-denied_table3|t
-denied_tableas3|t
-denied_view3|t
-denied_mview3|t
-denied_selectinto3|t
-1
-denied_function3|t
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-DROP FUNCTION
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-CREATE FUNCTION
-1
-1
-1
-1
-1
-1
-denied_function4|t
-1
-1
-1
-1
-1
-1
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-DROP FUNCTION
-

test/CDB_QuotaTest.sql

@@ -1,48 +1,30 @@
 set client_min_messages to error;
 \set VERBOSITY TERSE
--- Runs a query and returns whether an error was thrown
--- Useful when the error message depends on the execution plan or db settings
--- The error message outputs the extra quota, and this might depend on the database setup and version
-CREATE OR REPLACE FUNCTION catch_error(query text)
-RETURNS bool
-AS $$
-BEGIN
-    EXECUTE query;
-    RETURN FALSE;
-EXCEPTION
-    WHEN OTHERS THEN
-        RETURN TRUE;
-END
-$$ LANGUAGE 'plpgsql';
+
+-- See the dice
+SELECT setseed(0.5);
 
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-SELECT 'excess1', catch_error($$INSERT INTO big VALUES (2); $$); -- disallowed, quota exceeds before
-SELECT cartodb.CDB_SetUserQuotaInBytes(0);
-SELECT cartodb.CDB_CartodbfyTable('big');
--- Creating the trigger should fail as it was created by CDB_CartodbfyTable
-CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
--- Drop the trigger and recreate it forcing a 100% checks
-DROP TRIGGER test_quota ON big;
-CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+SELECT CDB_SetUserQuotaInBytes(0);
+SELECT CDB_CartodbfyTable('big');
 INSERT INTO big SELECT generate_series(2049,4096);
 INSERT INTO big SELECT generate_series(4097,6144);
 INSERT INTO big SELECT generate_series(6145,8192);
 -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
-SELECT cartodb.CDB_UserDataSize() < 500000 AND cartodb.CDB_UserDataSize() > 0;
-SELECT cartodb._CDB_total_relation_size('public', 'big') < 1000000;
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
 SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
 -- END Test for #108
-
-SELECT cartodb.CDB_SetUserQuotaInBytes(2);
-SELECT 'excess2', catch_error($$INSERT INTO big VALUES (8193);$$);
-SELECT cartodb.CDB_SetUserQuotaInBytes(0);
+SELECT setseed(0.9);
+SELECT CDB_SetUserQuotaInBytes(2);
+INSERT INTO big VALUES (8193);
+SELECT CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (8194);
 DROP TABLE big;
 
@@ -50,20 +32,16 @@ DROP TABLE big;
 --analysis tables should be excluded from quota:
 CREATE TABLE big(a int);
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
-SELECT cartodb.CDB_SetUserQuotaInBytes(1);
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+SELECT CDB_SetUserQuotaInBytes(1);
 CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int);
 INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5);
 INSERT INTO big VALUES (1); -- allowed, check runs before
 DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4;
-SELECT 'excess3', catch_error($$INSERT INTO big VALUES (3);$$); -- disallowed, quota exceeds before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
 DROP TABLE big;
 SELECT CDB_SetUserQuotaInBytes(0);
 
-CREATE SCHEMA "complex-name%_with'quotes""";
-SELECT CDB_UserDataSize('"complex-name%_with''quotes"');
-DROP SCHEMA "complex-name%_with'quotes""";
 
 set client_min_messages to NOTICE;
-DROP FUNCTION catch_error(text);
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,22 +1,20 @@
 SET
-CREATE FUNCTION
+
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
-excess1|t
+ERROR:  Quota exceeded by 3.9990234375KB
 0
 big
-ERROR:  trigger "test_quota" for relation "big" already exists
-DROP TRIGGER
-CREATE TRIGGER
 INSERT 0 2048
 INSERT 0 2048
 INSERT 0 2048
-t
-t
+454656
+909312
 0
+
 2
-excess2|t
+ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE
@@ -27,12 +25,8 @@ CREATE TABLE
 INSERT 0 5
 INSERT 0 1
 DROP TABLE
-excess3|t
+ERROR:  Quota exceeded by 3.9990234375KB
 DROP TABLE
 0
-CREATE SCHEMA
-0
-DROP SCHEMA
 SET
 DROP FUNCTION
-DROP FUNCTION

test/CDB_RegenerateTable.sql

@@ -1,292 +0,0 @@
--- Setup
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-SET SESSION AUTHORIZATION postgres;
-\set QUIET off
-
-\echo '## Setup'
-CREATE TABLE testtable (stable integer, c1 integer, c2 integer, c3 integer, c4 integer);
-INSERT INTO testtable(stable,c1,c2,c3,c4) VALUES (1,2,3,4,5), (2,3,4,5,6), (3,4,5,6,7);
-\d+ testtable
-SELECT * FROM testtable ORDER BY stable ASC;
-SELECT 'testtable'::regclass::oid as id INTO temp table original_oid;
-
-CREATE FUNCTION cartodb.CDB_GetTableQueries_TestHelper(tableoid OID, ignore_cartodbfication BOOL DEFAULT false)
-RETURNS text[]
-AS
-$$
-DECLARE
-    queries TEXT[];
-BEGIN
-    -- In older version of PG (pre 11), the syntax when creating triggers was
-    -- CREATE TRIGGER ... EXECUTE PROCEDURE ...
-    -- But in new ones it is
-    -- CREATE TRIGGER ... EXECUTE FUNCTION ...
-    -- To uniformize the tests, we replace it in the output of CDB_RegenerateTable
-
-    EXECUTE FORMAT('
-        SELECT array_agg(REGEXP_REPLACE(a, ''EXECUTE PROCEDURE'', ''EXECUTE FUNCTION''))
-            FROM unnest(cartodb.CDB_GetTableQueries(%L, %L)) a;', tableoid, ignore_cartodbfication) INTO queries;
-    RETURN queries;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-\echo '## Run cartodb.CDB_RegenerateTable and confirm the data and columns are the same'
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-\d+ testtable
-SELECT * FROM testtable ORDER BY stable ASC;
-
-\echo '## The table oid must have changed since the table itself changed'
-SELECT 'testtable'::regclass::oid as id INTO temp table new_oid;
-SELECT original_oid.id = new_oid.id FROM original_oid, new_oid;
-
-\echo '## Check adding an index'
-CREATE INDEX testtable_stable_idx ON testtable (stable NULLS FIRST) WITH (fillfactor = 80, vacuum_cleanup_index_scale_factor = 0.11);
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-
-\echo '## Check column properties'
-ALTER TABLE testtable ADD UNIQUE (c2);
-ALTER TABLE testtable ALTER COLUMN c3 SET NOT NULL;
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-\echo '## Check triggers'
-CREATE OR REPLACE FUNCTION trigger_example_fn()
-    RETURNS TRIGGER 
-    LANGUAGE PLPGSQL
-AS
-$$
-BEGIN
-	RETURN NEW;
-END;
-$$;
-
-CREATE TRIGGER testtable_trigger_example
-    BEFORE UPDATE
-    ON testtable
-    FOR EACH ROW
-    EXECUTE PROCEDURE trigger_example_fn();
-
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-\echo '## Check Cartodbfycation'
-DROP INDEX testtable_stable_idx;
-DROP TRIGGER testtable_trigger_example ON testtable;
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-SELECT CDB_SetUserQuotaInBytes(0);
-SELECT CDB_CartodbfyTable('testtable'::regclass);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-\echo '## Test view / matview dependencies: It will not work but data will be the same'
-CREATE VIEW testview AS SELECT * FROM testtable WHERE stable < 20;
-SELECT * FROM testview ORDER BY stable ASC;
-\d testtable
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-DROP VIEW testview;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-CREATE MATERIALIZED VIEW testmatview AS SELECT * FROM testtable WHERE stable < 20;
-SELECT * FROM testmatview ORDER BY stable ASC;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-DROP MATERIALIZED VIEW testmatview;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\d testtable
-
-\echo '## Test role access'
-CREATE ROLE cdb_regenerate_tester LOGIN PASSWORD 'cdb_regenerate_pass';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_regenerate_tester;
-GRANT SELECT ON testtable TO cdb_regenerate_tester;
-
-\c contrib_regression cdb_regenerate_tester
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT * FROM testtable ORDER BY cartodb_id DESC;
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\c contrib_regression cdb_regenerate_tester
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT * FROM testtable ORDER BY cartodb_id DESC;
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-\echo '## Test calling with read only access (should fail)'
-\c contrib_regression cdb_regenerate_tester
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-\echo '## Test partitioned table'
-CREATE TABLE measurement (
-    city_id         int not null,
-    logdate         date not null,
-    peaktemp        int,
-    unitsales       int
-) PARTITION BY RANGE (logdate);
-
-CREATE TABLE measurement_y2006m02 PARTITION OF measurement
-    FOR VALUES FROM ('2006-02-01') TO ('2006-03-01')
-    PARTITION BY RANGE (peaktemp);
-
-CREATE TABLE measurement_y2006m03 PARTITION OF measurement
-    FOR VALUES FROM ('2006-03-01') TO ('2006-04-01');
-CREATE INDEX ON measurement_y2006m02 (logdate);
-CREATE INDEX ON measurement_y2006m03 (logdate);
-
-\d measurement
-SELECT  c.oid::pg_catalog.regclass,
-        pg_catalog.pg_get_expr(c.relpartbound, c.oid),
-        c.relkind
-FROM pg_catalog.pg_class c,
-     pg_catalog.pg_inherits i
-WHERE c.oid=i.inhrelid AND i.inhparent = 'measurement'::regclass::oid
-ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text;
-\d measurement_y2006m02
-\d measurement_y2006m03
-
-SELECT cartodb.CDB_RegenerateTable('measurement'::regclass::oid);
-SELECT cartodb.CDB_RegenerateTable('measurement_y2006m02'::regclass::oid);
-SELECT  c.oid::pg_catalog.regclass,
-        pg_catalog.pg_get_expr(c.relpartbound, c.oid),
-        c.relkind
-FROM pg_catalog.pg_class c,
-     pg_catalog.pg_inherits i
-WHERE c.oid=i.inhrelid AND i.inhparent = 'measurement'::regclass::oid
-ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text;
-\d measurement_y2006m02
-\d measurement_y2006m03
-
-SELECT cartodb.CDB_GetTableQueries_TestHelper('measurement'::regclass::oid, ignore_cartodbfication := false);
-
-\echo '## Test transaction with truncate'
-SET statement_timeout = 1000;
-BEGIN;
-    TRUNCATE TABLE testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-COMMIT;
-
-\echo '## Test transaction with delete'
-BEGIN;
-    DELETE FROM testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-COMMIT;
-
-\echo '## Test transaction with delete + cartodbfy'
-BEGIN;
-    INSERT INTO testtable(stable,c1,c2,c3,c4) VALUES (1,2,3,4,5), (2,3,4,5,6), (3,4,5,6,7);
-    DELETE FROM testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-    SELECT CDB_CartodbfyTable('public'::TEXT, 'public.testtable'::REGCLASS);
-COMMIT;
-
-\echo '## Test replacement in import (drop c3 and c4 columns)'
-CREATE INDEX testtable_c4_idx ON testtable (c4 NULLS FIRST);
-
-\d testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-DO $$
-DECLARE
-    queries TEXT[] :=  CDB_GetTableQueries_TestHelper('testtable'::regclass, true);
-BEGIN
-    DROP TABLE testtable;
-    CREATE TABLE testtable (stable integer, c1 integer, c2 integer);
-    PERFORM CDB_CartodbfyTable('public.testtable');
-    PERFORM CDB_ApplyQueriesSafe(queries);
-END$$;
-
-\d testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-\echo '## teardown'
-
-DROP TABLE measurement CASCADE;
-DROP TABLE testtable CASCADE;
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_regenerate_tester;
-DROP ROLE cdb_regenerate_tester;
-DROP FUNCTION cartodb.CDB_GetTableQueries_TestHelper;

test/CDB_RegenerateTable_expect

@@ -1,228 +0,0 @@
-## Setup
-CREATE TABLE
-INSERT 0 3
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-1|2|3|4|5
-2|3|4|5|6
-3|4|5|6|7
-SELECT 1
-CREATE FUNCTION
-## Run cartodb.CDB_RegenerateTable and confirm the data and columns are the same
-
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-1|2|3|4|5
-2|3|4|5|6
-3|4|5|6|7
-## The table oid must have changed since the table itself changed
-SELECT 1
-f
-## Check adding an index
-CREATE INDEX
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-## Check column properties
-ALTER TABLE
-ALTER TABLE
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||not null||plain||
-c4|integer||||plain||
-testtable|testtable_c2_key|CREATE UNIQUE INDEX testtable_c2_key ON public.testtable USING btree (c2)
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||not null||plain||
-c4|integer||||plain||
-testtable|testtable_c2_key|CREATE UNIQUE INDEX testtable_c2_key ON public.testtable USING btree (c2)
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-## Check triggers
-CREATE FUNCTION
-CREATE TRIGGER
-public|testtable|public|testtable_trigger_example
-
-public|testtable|public|testtable_trigger_example
-## Check Cartodbfycation
-DROP INDEX
-DROP TRIGGER
-{"ALTER TABLE public.testtable OWNER TO postgres","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_c2_key UNIQUE (c2)"}
-{"ALTER TABLE public.testtable OWNER TO postgres","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_c2_key UNIQUE (c2)"}
-0
-testtable
-{"ALTER TABLE public.testtable OWNER TO postgres","CREATE SEQUENCE public.testtable_cartodb_id_seq    START WITH 1    INCREMENT BY 1    NO MINVALUE    NO MAXVALUE    CACHE 1","ALTER TABLE public.testtable_cartodb_id_seq OWNER TO postgres","ALTER SEQUENCE public.testtable_cartodb_id_seq OWNED BY public.testtable.cartodb_id","ALTER TABLE ONLY public.testtable ALTER COLUMN cartodb_id SET DEFAULT nextval('public.testtable_cartodb_id_seq'::regclass)","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_pkey PRIMARY KEY (cartodb_id)","CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)","CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)","CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_checkquota('0.1', '-1', 'public')","CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb.cdb_checkquota('0.001', '-1', 'public')","CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_tablemetadata_trigger()","CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb._cdb_update_the_geom_webmercator()"}
-{"ALTER TABLE public.testtable OWNER TO postgres"}
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)|plain||
-the_geom|geometry(Geometry,4326)||||main||
-the_geom_webmercator|geometry(Geometry,3857)||||main||
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-public|testtable|public|test_quota
-public|testtable|public|test_quota_per_row
-public|testtable|public|track_updates
-public|testtable|public|update_the_geom_webmercator_trigger
-
-{"ALTER TABLE public.testtable OWNER TO postgres","CREATE SEQUENCE public.testtable_cartodb_id_seq    START WITH 1    INCREMENT BY 1    NO MINVALUE    NO MAXVALUE    CACHE 1","ALTER TABLE public.testtable_cartodb_id_seq OWNER TO postgres","ALTER SEQUENCE public.testtable_cartodb_id_seq OWNED BY public.testtable.cartodb_id","ALTER TABLE ONLY public.testtable ALTER COLUMN cartodb_id SET DEFAULT nextval('public.testtable_cartodb_id_seq'::regclass)","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_pkey PRIMARY KEY (cartodb_id)","CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)","CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)","CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_checkquota('0.1', '-1', 'public')","CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb.cdb_checkquota('0.001', '-1', 'public')","CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_tablemetadata_trigger()","CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb._cdb_update_the_geom_webmercator()"}
-{"ALTER TABLE public.testtable OWNER TO postgres"}
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)|plain||
-the_geom|geometry(Geometry,4326)||||main||
-the_geom_webmercator|geometry(Geometry,3857)||||main||
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-public|testtable|public|test_quota
-public|testtable|public|test_quota_per_row
-public|testtable|public|track_updates
-public|testtable|public|update_the_geom_webmercator_trigger
-## Test view / matview dependencies: It will not work but data will be the same
-CREATE VIEW
-1|||1|2|3|4|5
-2|||2|3|4|5|6
-3|||3|4|5|6|7
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-ERROR:  cannot drop table testtable because other objects depend on it
-DROP VIEW
-
-SELECT 3
-1|||1|2|3|4|5
-2|||2|3|4|5|6
-3|||3|4|5|6|7
-ERROR:  cannot drop table testtable because other objects depend on it
-DROP MATERIALIZED VIEW
-
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-## Test role access
-CREATE ROLE
-GRANT
-GRANT
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-3|||3|4|5|6|7
-2|||2|3|4|5|6
-1|||1|2|3|4|5
-You are now connected to database "contrib_regression" as user "postgres".
-
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-3|||3|4|5|6|7
-2|||2|3|4|5|6
-1|||1|2|3|4|5
-You are now connected to database "contrib_regression" as user "postgres".
-## Test calling with read only access (should fail)
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-ERROR:  must be owner of table testtable
-You are now connected to database "contrib_regression" as user "postgres".
-## Test partitioned table
-CREATE TABLE
-CREATE TABLE
-CREATE TABLE
-CREATE INDEX
-CREATE INDEX
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-measurement_y2006m02|FOR VALUES FROM ('02-01-2006') TO ('03-01-2006')|p
-measurement_y2006m03|FOR VALUES FROM ('03-01-2006') TO ('04-01-2006')|r
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-ERROR:  CDB_RegenerateTable does not support the parent of partitioned tables
-
-measurement_y2006m02|FOR VALUES FROM ('02-01-2006') TO ('03-01-2006')|p
-measurement_y2006m03|FOR VALUES FROM ('03-01-2006') TO ('04-01-2006')|r
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-ERROR:  CDB_GetTableQueries does not support the parent of partitioned tables
-## Test transaction with truncate
-SET
-BEGIN
-TRUNCATE TABLE
-ERROR:  plpy.Error: Could not get table properties
-ROLLBACK
-## Test transaction with delete
-BEGIN
-DELETE 3
-
-COMMIT
-## Test transaction with delete + cartodbfy
-BEGIN
-INSERT 0 3
-DELETE 3
-
-testtable
-COMMIT
-## Test replacement in import (drop c3 and c4 columns)
-CREATE INDEX
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-testtable|testtable_c4_idx|CREATE INDEX testtable_c4_idx ON public.testtable USING btree (c4 NULLS FIRST)
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-DO
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-## teardown
-DROP TABLE
-DROP TABLE
-REVOKE
-DROP ROLE
-DROP FUNCTION

test/CDB_SyncTableTest.sql

@@ -51,7 +51,7 @@ UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecodin
 SET client_min_messages TO notice;
 SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}');
 SELECT * FROM test_sync_source ORDER BY cartodb_id;
-SELECT cartodb_id, the_geom, lat, lon, name FROM test_sync_dest ORDER BY cartodb_id;
+SELECT * FROM test_sync_dest ORDER BY cartodb_id;
 
 \echo 'It will work with schemas that need quoting'
 \set QUIET on

test/CDB_SyncTableTest_expect

@@ -54,10 +54,10 @@ NOTICE:  MODIFIED 0 row(s)
 2|||2|2|bar
 4|||4|4|cantaloupe
 5|||5|5|sandia
-1|0101000020E6100000000000000000F03F000000000000F03F|1|1|foo
-2|0101000020E610000000000000000000400000000000000040|2|2|bar
-4|0101000020E610000000000000000010400000000000001040|4|4|cantaloupe
-5|0101000020E610000000000000000014400000000000001440|5|5|sandia
+1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo
+2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar
+4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe
+5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia
 It will work with schemas that need quoting
 
 INSERT 0 1

test/CDB_UserTablesTest.sql

@@ -17,16 +17,16 @@ CREATE TABLE pub(a int);
 CREATE TABLE prv(a int);
 GRANT SELECT ON TABLE pub TO publicuser;
 REVOKE SELECT ON TABLE prv FROM publicuser;
-SELECT cartodb.CDB_UserTables() ORDER BY 1;
-SELECT 'all', cartodb.CDB_UserTables('all') ORDER BY 2;
-SELECT 'public', cartodb.CDB_UserTables('public') ORDER BY 2;
-SELECT 'private', cartodb.CDB_UserTables('private') ORDER BY 2;
-SELECT '--unsupported--', cartodb.CDB_UserTables('--unsupported--') ORDER BY 2;
+SELECT CDB_UserTables() ORDER BY 1;
+SELECT 'all',CDB_UserTables('all') ORDER BY 2;
+SELECT 'public',CDB_UserTables('public') ORDER BY 2;
+SELECT 'private',CDB_UserTables('private') ORDER BY 2;
+SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2;
 -- now tests with public user
 \c contrib_regression publicuser
-SELECT 'all_publicuser', cartodb.CDB_UserTables('all') ORDER BY 2;
-SELECT 'public_publicuser', cartodb.CDB_UserTables('public') ORDER BY 2;
-SELECT 'private_publicuser', cartodb.CDB_UserTables('private') ORDER BY 2;
+SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2;
+SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2;
+SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2;
 \c contrib_regression postgres
 DROP TABLE pub;
 DROP TABLE prv;

test/CDB_Username_expect

@@ -1,4 +1,4 @@
-@@PGUSER@@
+postgres
 
 fulano
 fulanito

test/README

@@ -6,7 +6,6 @@ Example, to add a test for CDB_Something function, you'd add:
  - CDB_SomethingTest.sql
  - CDB_SomethingTest_expect
 
-
 To easy the generation of the expected file you can initially omit it,
 then run "make -C .. installcheck" from the top-level dir and copy
 ../results/test/CDB_SomethingTest.out to CDB_SomethingTest_expect chopping

test/extension/test.sh

@@ -17,10 +17,6 @@ SED=sed
 OK=0
 PARTIALOK=0
 
-function reset_default_database() {
-    DATABASE=test_extension
-}
-
 function set_failed() {
     OK=1
     PARTIALOK=1
@@ -44,10 +40,10 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as '${ROLE}' in '${DATABASE}'"
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
       RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
     else
-      log_debug "Executing query '${QUERY}' in '${DATABASE}'"
+      log_debug "Executing query '${QUERY}'"
       RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
     fi
     CODERESULT=$?
@@ -216,8 +212,6 @@ function tear_down_database() {
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
 function tear_down() {
-    reset_default_database
-
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
     sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
@@ -368,7 +362,7 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
 
 function test_cdb_tablemetadatatouch_fully_qualifies_names() {
     sql postgres "CREATE TABLE touch_invalidations (table_name text);"
-    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into public.touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
 
     #default schema
     sql "CREATE TABLE touch_example (a int);"
@@ -538,8 +532,6 @@ END
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
 
-    reset_default_database
-
     # Add PGPORT to conf if it is set
     PORT_SPEC=""
     if [[ "$PGPORT" != "" ]] ; then
@@ -598,6 +590,38 @@ test_extension|public|"local-table-with-dashes"'
     sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;"
     sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't'
 
+
+    # Check user-defined FDW's
+
+    # Grant the user permissions to use the postgres_fdw
+    sql postgres "GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO cdb_testmember_1;"
+    sql postgres "ALTER ROLE cdb_testmember_1 WITH CREATEROLE;"
+
+    # Set up a user foreign server
+    read -d '' ufdw_config <<- EOF
+{
+   "server": {
+     "extensions": "postgis",
+     "dbname": "fdw_target",
+     "host": "localhost",
+     "port": ${PGPORT:-5432}
+   },
+   "user_mapping": {
+     "user": "fdw_user",
+     "password": "foobarino"
+   }
+}
+EOF
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_Foreign_Server('test_user_fdw', '$ufdw_config');"
+
+    # Set up a user foreign table
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_Foreign_Table('test_user_fdw', 'test_fdw', 'foo');"
+
+    # Check that the table can be accessed
+    sql cdb_testmember_1 "SELECT * from test_user_fdw.foo;"
+    sql cdb_testmember_1 "SELECT a from test_user_fdw.foo LIMIT 1;" should 42
+
+
     # Teardown
     DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
@@ -605,10 +629,16 @@ test_extension|public|"local-table-with-dashes"'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
 
-    reset_default_database
+    # TODO add to function to delete stuff
+    sql postgres 'DROP FOREIGN TABLE test_user_fdw.foo;'
+    sql postgres 'DROP schema test_user_fdw;'
+    sql postgres 'DROP USER MAPPING FOR public SERVER test_user_fdw;'
+    sql postgres 'DROP SERVER test_user_fdw;'
+    sql postgres 'REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM test_user_fdw;'
+    sql postgres 'DROP ROLE test_user_fdw;'
+
     sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
     DATABASE=fdw_target tear_down_database
-    reset_default_database
 }
 
 function test_cdb_catalog_basic_node() {

util/create_from_unpackaged.sh

@@ -10,7 +10,6 @@ echo "-- Script generated by $0 on `date`" > ${output}
 cat ${input} |
   grep '^ *CREATE OR REPLACE FUNCTION' |
   grep -v ' cartodb\.' | # should  only match DDL hooks
-  grep -v '.*\quit.*' | 
   sed 's/).*$/)/' |
   sed 's/DEFAULT [^ ,)]*//g' |
   sed 's/CREATE OR REPLACE FUNCTION /ALTER FUNCTION public./' |
@@ -20,4 +19,59 @@ cat ${input} |
   cat >> ${output}
 
 # Upgrade all functions
-cat ${input} | grep -v 'duplicated extension$' | grep -v '\quit$' | grep -v 'pg_extension_config_dump' >> ${output}
+cat ${input} | grep -v 'duplicated extension$' >> ${output}
+
+# Migrate CDB_TableMetadata
+cat >> ${output} <<'EOF'
+ALTER TABLE cartodb.CDB_TableMetadata DISABLE TRIGGER ALL;
+INSERT INTO cartodb.CDB_TableMetadata SELECT * FROM public.CDB_TableMetadata;
+ALTER TABLE cartodb.CDB_TableMetadata ENABLE TRIGGER ALL;
+DROP TABLE public.CDB_TableMetadata;
+
+-- Set user quota
+-- NOTE: will fail if user quota wasn't set at database level, see
+--       http://github.com/CartoDB/cartodb-postgresql/issues/18
+DO $$
+DECLARE
+  qmax int8;
+BEGIN
+  BEGIN
+    qmax := public._CDB_UserQuotaInBytes();
+  EXCEPTION WHEN undefined_function THEN
+    RAISE EXCEPTION 'Please set user quota before switching to cartodb extension';
+  END;
+  PERFORM cartodb.CDB_SetUserQuotaInBytes(qmax);
+  DROP FUNCTION public._CDB_UserQuotaInBytes();
+END;
+$$ LANGUAGE 'plpgsql';
+EOF
+
+## Cartodbfy tables with a trigger using 'CDB_CheckQuota' or
+## 'CDB_TableMetadata_Trigger' from the 'public' schema
+#cat >> ${output} <<'EOF'
+#select cartodb.CDB_CartodbfyTable(relname::regclass) from ( 
+#  -- names of tables using public.CDB_CheckQuota or
+#  -- public.CDB_TableMetadata_Trigger in their triggers
+#  SELECT distinct c.relname
+#  FROM
+#    pg_trigger t,
+#    pg_class c,
+#    pg_proc p,
+#    pg_namespace n
+#  WHERE
+#    n.nspname = 'public' AND
+#    p.pronamespace = n.oid AND
+#    p.proname IN ( 'cdb_checkquota', 'cdb_tablemetadata_trigger' ) AND
+#    t.tgrelid = c.oid AND
+#    p.oid = t.tgfoid 
+#) as foo;
+#EOF
+
+## Drop any leftover function from public schema (there should be none)
+#cat ${input} |
+#  grep '^ *CREATE OR REPLACE FUNCTION' |
+#  grep -v ' cartodb\.' | # should  only match DDL hooks
+#  sed 's/).*$/);/' |
+#  sed 's/DEFAULT [^ ,)]*//g' |
+#  sed 's/CREATE OR REPLACE FUNCTION /DROP FUNCTION IF EXISTS public./' |
+#  cat >> ${output}