CDB_MapConfig table and triggers script

.travis.yml

@@ -9,23 +9,24 @@ env:
 
 jobs:
   include:
+    - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+      dist: xenial
+    - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+      dist: xenial
     - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
       dist: xenial
     - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
       dist: bionic
     - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
       dist: bionic
-    - env: POSTGRESQL_VERSION="13" POSTGIS_VERSION="3"
-      dist: bionic
 
 script:
-  - sudo apt-get remove postgresql* -y
   - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
   - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
   - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
   # For pre12, install plpython2. For PG12 install plpython3
-  - if [[ $POSTGRESQL_VERSION == '11' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-$POSTGRESQL_VERSION python3 python3-redis; fi;
-  - for i in $(pg_lsclusters  | tail -n +2 | awk '{print $1}'); do sudo pg_dropcluster --stop $i main; done;
+  - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi;
+  - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
   - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION
   - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -- --auth-local trust --auth-host password
   - export PGPORT=$(pg_lsclusters | grep $POSTGRESQL_VERSION | awk '{print $3}')
@@ -38,4 +39,4 @@ after_failure:
   - cat regression.out
   - cat regression.diffs
   - echo $PGPORT
-  - sudo cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log
+  - cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.37.1
+EXTVERSION = 0.36.0
 
 SED = sed
 AWK = awk
@@ -109,7 +109,6 @@ UPGRADABLE = \
   0.34.0 \
   0.35.0 \
   0.36.0 \
-  0.37.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -120,6 +119,8 @@ UPGRADES = \
      $(SED) 's/$$/--$(EXTVERSION).sql/' | \
      $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g')
 
+GITDIR=$(shell test -d .git && echo '.git' || cat .git | $(SED) 's/^gitdir: //')
+
 DATA_built = \
   $(EXTENSION)--$(EXTVERSION).sql \
   $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql \
@@ -167,7 +168,7 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 $(EXTENSION).control: $(EXTENSION).control.in Makefile
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
 
-cartodb_version.sql: cartodb_version.sql.in Makefile
+cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
 
 # Needed for consistent `echo` results with backslashes

NEWS.md

@@ -1,14 +1,3 @@
-0.37.1 (2020-12-02)
-* Change `__CDB_RegenerateTable_Get_Commands` to use the caller timeout or '1min' if not set.
-
-0.37.0 (2020-11-26)
-* Raised minimum PG version to 11.
-* Add `CDB_RegenerateTable` function to regenerate a table.
-* Add `CDB_GetTableQueries` to get the queries of a table (constraints, indices, triggers...).
-* Add `CDB_ApplyQueriesSafe` to apply the queries of `CDB_GetTableQueries` discarding any exceptions.
-* Deprecate creation of new overview tables.
-* _cdb_has_usable_geom_record: Check only the extension schema.
-
 0.36.0 (2020-02-13)
 * Make `_CDB_Group_API_Auth` python3 compatible by passing bytes representation instead of a string.
 * Make `_CDB_Group_API_Request` python3 compatible by adapting the function signature of `HTTPConnection`.

README.md

@@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/
 Dependencies
 ------------
 
- * PostgreSQL 11+ (with plpythonu extension). For PostgreSQL 12+ plpython3u is required instead. Older versions might still work but they aren't actively tested or supported.
+ * PostgreSQL 9.6+ (with plpythonu extension and xml support). For PostgreSQL 12+ plpython3u is required instead of plpythonu.
  * [PostGIS extension](http://postgis.net)
  * Python with [Redis module](https://pypi.org/project/redis/)
 
@@ -30,7 +30,7 @@ make installcheck
 
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
-
+      
 NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
@@ -39,7 +39,23 @@ Enable database
 In a database that needs to be turned into a "cartodb" user database, run:
 
 ```sql
-CREATE EXTENSION cartodb CASCADE;
+CREATE EXTENSION postgis;
+CREATE EXTENSION cartodb;
+```
+
+Migrate existing cartodb database
+---------------------------------
+
+When upgrading an existing cartodb user database, the cartodb extension
+can be migrated from the "unpackaged" version. The procedure will copy
+the data from ``public.CDB_TableMetada`` to ``cartodb.CDB_TableMetadata``,
+re-cartodbfy all tables using old functions in triggers and drop the
+cartodb functions from the 'public' schema. All new cartodb objects will
+be in the "cartodb" schema.
+
+```sql
+CREATE EXTENSION postgis FROM unpackaged;
+CREATE EXTENSION cartodb FROM unpackaged;
 ```
 
 Update cartodb extension

doc/CDB_Overviews.md

@@ -1,8 +1,3 @@
-
-============================ WARNING ===================================
-Creating overviews is no longer supported.
-============================ WARNING ===================================
-
 Overviews are tables that represent a *reduced* version of a dataset intended
 for efficient rendering at certain zoom levels while preserving the
 general visual appearance of the complete dataset.

scripts-available/CDB_CartodbfyTable.sql

@@ -636,7 +636,8 @@ $$ LANGUAGE 'plpgsql' STABLE PARALLEL SAFE;
 
 DO $$
 BEGIN
-  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record' AND typnamespace = '@extschema@'::regnamespace) THEN
+  SET search_path TO @extschema@;
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record') THEN
     CREATE TYPE @extschema@._cdb_has_usable_geom_record
       AS (has_usable_geoms boolean,
         text_geom_column boolean,
@@ -845,7 +846,7 @@ BEGIN
   -- values. No usable pk implies has_usable_pk_sequence = false.
   has_usable_pk_sequence := false;
   IF has_usable_primary_key THEN
-    SELECT @extschema@._CDB_Has_Usable_PK_Sequence(reloid)
+    SELECT _CDB_Has_Usable_PK_Sequence(reloid)
     INTO STRICT has_usable_pk_sequence;
   END IF;
 

scripts-available/CDB_MapConfig.sql

@@ -0,0 +1,57 @@
+-- Table to save map configs
+CREATE TABLE IF NOT EXISTS
+@extschema@.CDB_MapConfig (
+  id char(32) not null primary key,
+  map jsonb not null, 
+  used_at timestamp with time zone NOT NULL DEFAULT now()
+);
+
+-- trigger function which updates the last day date 
+-- only when the current date is older than 1 day
+CREATE OR REPLACE FUNCTION @extschema@._CDB_MapConfig_Update_Used_At()
+  RETURNS trigger 
+AS $$
+BEGIN
+  IF NEW."used_at" < (now() - '1 days'::interval) THEN
+    NEW."used_at" = NOW();
+    RETURN NEW;
+  END IF;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+--  create trigger
+DROP TRIGGER IF EXISTS update_used_at ON @extschema@.CDB_MapConfig;
+CREATE TRIGGER update_used_at 
+AFTER SELECT ON @extschema@.CDB_MapConfig 
+FOR EACH ROW EXECUTE PROCEDURE @extschema@._CDB_MapConfig_Update_Used_At();
+
+
+-- trigger function which removes the latest map config
+-- if the date is older than 30 days
+CREATE OR REPLACE FUNCTION @extschema@._CDB_MapConfig_Remove_Latest_One()
+  RETURNS trigger 
+AS $$
+BEGIN
+  SELECT id 
+  FROM @extschema@.CDB_MapConfig
+  WHERE used_at < (now() - '30 days'::interval)
+  ORDER BY used_at ASC
+  LIMIT 1 
+  INTO map_id;
+
+  IF map_id IS NOT NULL THEN
+    DELETE FROM @extschema@.CDB_MapConfig
+    WHERE id = map_id;
+  END IF;
+
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+--  create trigger
+DROP TRIGGER IF EXISTS remove_latest_one ON @extschema@.CDB_MapConfig;
+CREATE TRIGGER remove_latest_one 
+BEFORE INSERT ON @extschema@.CDB_MapConfig 
+FOR EACH STATEMENT EXECUTE PROCEDURE @extschema@._CDB_MapConfig_Remove_Latest_One();
+
+
+

scripts-available/CDB_Overviews.sql

@@ -268,7 +268,44 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
+    fraction := power(2, 2*(overview_z - ref_z));
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Sampling reduce stategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Sampling reduce stategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Estimate number of rows
+    SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
+      WHERE oid = reloid::oid;
+
+    IF class_info.relpages < 2 OR fraction > 0.5 THEN
+      -- We'll avoid possible CDB_RandomTids problems
+      EXECUTE Format('
+        %s SELECT * FROM %s WHERE random() < %s;
+      ', creation_clause, reloid, fraction);
+    ELSE
+      num_samples := ceil(class_info.reltuples*fraction);
+      EXECUTE Format('
+        %1$s SELECT * FROM %2$s
+          WHERE ctid = ANY (
+            ARRAY[
+              (SELECT @extschema@.CDB_RandomTids(''%2$s'', %3$s))
+            ]
+          );
+      ', creation_clause, reloid, num_samples);
+    END IF;
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -294,7 +331,36 @@ AS $$
     overview_scheme TEXT;
     overview_name TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    -- This function will only register a table as an overview table if it matches
+    -- the overviews naming scheme for the dataset and z level and the table belongs
+    -- to the same scheme as the the dataset
+    SELECT * FROM @extschema@._cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name;
+    SELECT * FROM @extschema@._cdb_split_table_name(overview_table) INTO overview_scheme, overview_name;
+    IF dataset_scheme = overview_scheme AND
+       overview_name = @extschema@._CDB_OverviewTableName(dataset_name, overview_z) THEN
+
+      -- preserve the owner of the base table
+      SELECT u.usename
+        FROM pg_catalog.pg_class c
+          JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
+          JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = dataset_name::text AND n.nspname = dataset_scheme
+        INTO table_owner;
+
+      EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
+
+      -- preserve the table privileges
+      UPDATE pg_class c_to
+        SET  relacl = c_from.relacl
+        FROM  pg_class c_from
+        WHERE c_from.oid  = dataset
+        AND   c_to.oid    = overview_table;
+
+      PERFORM @extschema@._CDB_Add_Indexes(overview_table);
+
+      -- TODO: If metadata about existing overviews is to be stored
+      -- it should be done here (CDB_Overviews would consume such metadata)
+    END IF;
   END
 $$  LANGUAGE PLPGSQL
     VOLATILE
@@ -551,7 +617,105 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    SELECT @extschema@._CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT @extschema@.CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := @extschema@._CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := @extschema@._CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        WHEN 'the_geom' THEN
+          Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom)
+        WHEN 'the_geom_webmercator' THEN
+           Format('%s AS the_geom_webmercator', point_geom)
+        ELSE c::text
+        END AS column
+        FROM @extschema@.CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and selecte the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+      %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             Floor(@postgisschema@.ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(@postgisschema@.ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s f
+          WHERE f.the_geom_webmercator IS NOT NULL
+          GROUP BY gx, gy
+         )
+         SELECT %6$s FROM clusters
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -708,7 +872,104 @@ AS $$
     overview_table_name TEXT;
     creation_clause TEXT;
   BEGIN
-    RAISE EXCEPTION 'Creating overviews is deprecated';
+    SELECT @extschema@._CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := @extschema@._CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM @extschema@._cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT @extschema@.CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := @extschema@._CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := @extschema@._CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        ELSE c::text
+        END AS column
+        FROM @extschema@.CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster sampling strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster sampling strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and select the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+       %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             Floor(@postgisschema@.ST_X(_f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(@postgisschema@.ST_Y(_f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s _f
+          GROUP BY gx, gy
+         ),
+         cluster_geom AS (
+           SELECT the_geom, the_geom_webmercator, clusters.*
+             FROM clusters INNER JOIN %1$s _g ON (clusters.cartodb_id = _g.cartodb_id)
+         )
+         SELECT %6$s FROM cluster_geom
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -729,7 +990,9 @@ AS $$
 DECLARE
   tolerance_px FLOAT8;
 BEGIN
-  RAISE EXCEPTION 'Creating overviews is deprecated';
+  -- Use the default tolerance
+  tolerance_px := 1.0;
+  RETURN @extschema@.CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
 END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
@@ -748,7 +1011,54 @@ DECLARE
   has_counter_column boolean;
   has_overviews_for_z boolean;
 BEGIN
-  RAISE EXCEPTION 'Creating overviews is deprecated';
+  -- Determine the referece zoom level
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
+
+  IF ref_z < 0 OR ref_z IS NULL THEN
+    RETURN NULL;
+  END IF;
+
+  -- Determine overlay zoom levels
+  -- TODO: should be handled by the refscale_strategy?
+  overview_z := ref_z - 1;
+  WHILE overview_z >= 0 LOOP
+    SELECT array_append(overviews_z, overview_z) INTO overviews_z;
+    overview_z := overview_z - overviews_step;
+  END LOOP;
+
+  --  TODO Check for non-used overview to delete them but we have to be aware that the
+  --       current queries, for example from a tiler, are been used with the old overviews
+  --       so if we remove the overviews in the process this could lead to errors
+
+  -- Create or reganerate overlay tables
+  base_z := ref_z;
+  base_rel := reloid;
+  FOREACH overview_z IN ARRAY overviews_z LOOP
+    SELECT CASE WHEN count(*) > 0 THEN TRUE ELSE FALSE END from CDB_Overviews(reloid) WHERE z = overview_z INTO has_overviews_for_z;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s, ''%s'');', base_rel, base_z, overview_z, tolerance_px, has_overviews_for_z) INTO base_rel;
+    IF base_rel IS NULL THEN
+      EXIT;
+    END IF;
+    base_z := overview_z;
+    IF NOT has_overviews_for_z THEN
+      RAISE NOTICE 'Registering overview: %', base_rel;
+      PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
+    END IF;
+    SELECT array_append(overview_tables, base_rel) INTO overview_tables;
+  END LOOP;
+
+  IF overview_tables IS NOT NULL AND array_length(overview_tables, 1) > 0 THEN
+    SELECT EXISTS (
+      SELECT * FROM @extschema@.CDB_ColumnNames(reloid)  as colname WHERE colname = '_feature_count'
+    ) INTO has_counter_column;
+    IF NOT has_counter_column THEN
+      EXECUTE Format('
+        ALTER TABLE %s ADD COLUMN _feature_count integer DEFAULT 1;
+      ', reloid);
+    END IF;
+  END IF;
+
+  RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 

scripts-available/CDB_RegenerateTable.sql

@@ -1,153 +0,0 @@
---
--- Given a table, returns a series of queries that can be used to recreate it
--- It does not include data
---
-CREATE OR REPLACE FUNCTION @extschema@.__CDB_RegenerateTable_Get_Commands(tableoid OID)
-RETURNS text[]
-AS $$
-    import subprocess
-    import re
-
-    query = "SELECT current_setting('statement_timeout') as t"
-    rv = plpy.execute(query, 1)
-    timeout_string = str(rv[0]['t'])
-    if timeout_string == "0":
-        timeout_string = "1min"
-
-    query = "SELECT current_database()::text as dname"
-    rv = plpy.execute(query, 1)
-    database_name_string = str(rv[0]['dname'])
-
-    query = """SELECT concat(quote_ident(nspname), '.', quote_ident(relname)) as quoted_name
-                        FROM pg_catalog.pg_class AS c
-                        JOIN pg_catalog.pg_namespace AS ns
-                        ON c.relnamespace = ns.oid
-                        WHERE c.oid = '%s'""" % (tableoid)
-    rv = plpy.execute(query, 1)
-    full_tablename_string = str(rv[0]['quoted_name'])
-
-    # NOTE: We always use -s so data is never dumped!
-    # That would be a security issue that we would need to deal with (and we currently do not need it)
-    process_parameters = ["pg_dump", "-s", "--lock-wait-timeout", timeout_string, "-t", full_tablename_string, database_name_string]
-
-    proc = subprocess.Popen(process_parameters, stdout=subprocess.PIPE, shell=False)
-    (out, err) = proc.communicate()
-    if (err or not out):
-        plpy.error('Could not get table properties')
-
-    line = out.decode("utf-8")
-    lines = line.rsplit(";\n", -1)
-    clean_lines = []
-    for i in range(0, len(lines)):
-        line = lines[i]
-        sublines = line.splitlines()
-        sublines = [line.rstrip() for line in sublines]
-        sublines = [line for line in sublines if line]
-        sublines = [line for line in sublines if not line.startswith('--')]
-        sublines = [line for line in sublines if not line.lower().startswith('set ')]
-        sublines = [line for line in sublines if line.lower().find('pg_catalog.set_config(') == -1]
-
-        if len(sublines):
-            clean_lines.append("".join(sublines))
-
-    return clean_lines
-$$
-LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
-
--- Returns a list of queries that can be used to regenerate the structure of a table
--- The query to create the table is not included
--- The optional parameter **ignore_cartodbfication** will remove queries related to the cartodbfication of the table
-CREATE OR REPLACE FUNCTION @extschema@.CDB_GetTableQueries(tableoid OID, ignore_cartodbfication BOOL DEFAULT false)
-RETURNS text[]
-AS
-$$
-DECLARE
-    children INTEGER;
-    queries TEXT[];
-BEGIN
-    EXECUTE FORMAT ('SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhparent =  %L', tableoid)
-            INTO children;
-    IF children > 0 THEN
-        RAISE EXCEPTION 'CDB_GetTableQueries does not support the parent of partitioned tables';
-    END IF;
-    IF NOT ignore_cartodbfication THEN
-        EXECUTE FORMAT('
-            SELECT array_agg(a)
-                FROM unnest(@extschema@.__CDB_RegenerateTable_Get_Commands(%L)) a
-                WHERE   a NOT SIMILAR TO ''CREATE TABLE%%'';', tableoid) INTO queries;
-    ELSE
-        EXECUTE FORMAT('
-            SELECT array_agg(a)
-                FROM unnest(@extschema@.__CDB_RegenerateTable_Get_Commands(%L)) a
-                WHERE   a NOT SIMILAR TO ''CREATE TABLE%%'' AND
-                        a NOT SIMILAR TO (''%%PRIMARY KEY \(cartodb_id\)%%'') AND
-                        a NOT SIMILAR TO (''%%cartodb_id_seq%%'') AND
-                        a NOT SIMILAR TO (''%%track_updates%%'') AND
-                        a NOT SIMILAR TO (''%%update_the_geom_webmercator_trigger%%'') AND
-                        a NOT SIMILAR TO (''%%test_quota%%'') AND
-                        a NOT SIMILAR TO (''%%test_quota_per_row%%'') AND
-                        a NOT SIMILAR TO (''%%gist \(the_geom\)%%'') AND
-                        a NOT SIMILAR TO (''%%gist \(the_geom_webmercator\)%%'');', tableoid) INTO queries;
-    END IF;
-    RETURN queries;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
--- Helper function to apply the result of CDB_GetTableQueries catching and discarding any exceptions
-CREATE OR REPLACE FUNCTION @extschema@.CDB_ApplyQueriesSafe(queries TEXT[])
-RETURNS void
-AS
-$$
-DECLARE
-    i INTEGER;
-BEGIN
-    IF array_length(queries, 1) > 0 THEN
-        FOR i IN 1 .. array_upper(queries, 1)
-        LOOP
-            BEGIN
-                EXECUTE queries[i];
-            EXCEPTION WHEN OTHERS THEN
-                CONTINUE;
-            END;
-        END LOOP;
-    END IF;
-END
-$$
-LANGUAGE PLPGSQL STRICT VOLATILE PARALLEL UNSAFE;
-
--- Regenerates a table
-CREATE OR REPLACE FUNCTION @extschema@.CDB_RegenerateTable(tableoid OID)
-RETURNS void
-AS
-$$
-DECLARE
-    temp_name TEXT := 'temp_' || encode(sha224(random()::text::bytea), 'hex');
-    table_name TEXT;
-    queries TEXT[] := @extschema@.__CDB_RegenerateTable_Get_Commands(tableoid);
-    i INTEGER;
-    children INTEGER;
-BEGIN
-    EXECUTE FORMAT ('SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhparent =  %L', tableoid)
-            INTO children;
-    IF children > 0 THEN
-        RAISE EXCEPTION 'CDB_RegenerateTable does not support the parent of partitioned tables';
-    END IF;
-
-    EXECUTE FORMAT('SELECT concat(quote_ident(nspname), ''.'', quote_ident(relname)) as quoted_name
-                        FROM pg_catalog.pg_class AS c
-                        JOIN pg_catalog.pg_namespace AS ns
-                        ON c.relnamespace = ns.oid
-                        WHERE c.oid = %L', tableoid) INTO table_name;
-
-    EXECUTE FORMAT('CREATE TEMPORARY TABLE %s ON COMMIT DROP AS SELECT * FROM %s', temp_name, table_name);
-    EXECUTE FORMAT('DROP TABLE %s', table_name);
-
-    FOR i IN 1 .. array_upper(queries, 1)
-    LOOP
-        EXECUTE queries[i];
-    END LOOP;
-
-    EXECUTE FORMAT('INSERT INTO %s SELECT * FROM %I', table_name, temp_name);
-END
-$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_TableIndexes.sql

@@ -16,7 +16,7 @@ AS $$
   JOIN pg_class
   ON pg_class.oid = idx.indexrelid 
   WHERE pg_indexes.tablename = '' || $1 || ''
-  AND '' || $1 || '' IN (SELECT @extschema@.CDB_UserTables())
+  AND '' || $1 || '' IN (SELECT CDB_UserTables())
   AND pg_class.relname=pg_indexes.indexname
   ;
 

scripts-available/CDB_TransformToWebmercator.sql

@@ -36,7 +36,7 @@ BEGIN
 
   -- Don't bother clipping if the geometry boundary doesn't
   -- go outside the valid extent.
-  IF @postgisschema@.geometry_within(latlon_input, valid_extent) THEN
+  IF latlon_input @ valid_extent THEN
     BEGIN
       RETURN @postgisschema@.ST_Transform(latlon_input, 3857);
     EXCEPTION WHEN OTHERS THEN

scripts-enabled/500-CDB_RegenerateTable.sql

@@ -1 +0,0 @@
-../scripts-available/CDB_RegenerateTable.sql

test/CDB_GreatCircle.sql

@@ -1,2 +1,2 @@
-select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)), 3);
-select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000), 3);
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)));
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000));

test/CDB_GreatCircle_expect

@@ -1,2 +1,2 @@
-LINESTRING(4.259 55.858,5.533 56.001,6.817 56.13,8.109 56.246,9.408 56.349,10.713 56.437,12.025 56.513,13.341 56.574,14.661 56.621,15.983 56.654,17.308 56.674,18.633 56.679,19.958 56.67,21.282 56.647,22.604 56.61,23.923 56.559,25.238 56.494,26.548 56.415,27.852 56.323,29.149 56.217,30.439 56.097,31.72 55.964,32.992 55.818,34.254 55.659,35.505 55.487,36.745 55.302,37.974 55.105,39.189 54.896,40.392 54.674,41.582 54.441,42.758 54.197,43.919 53.941,45.066 53.674,46.199 53.396,47.316 53.107,48.418 52.809,49.505 52.5,50.577 52.181,51.633 51.853,52.673 51.515,53.698 51.169,54.708 50.813,55.702 50.449,56.681 50.077,57.644 49.697,58.593 49.308,59.526 48.912,60.445 48.509,61.349 48.098,62.238 47.681,63.114 47.257,63.975 46.826,64.822 46.388,65.656 45.945,66.477 45.496,67.284 45.041,68.079 44.58,68.861 44.114,69.631 43.642,70.389 43.166,71.135 42.684,71.869 42.198,72.592 41.708,73.304 41.212,74.006 40.713)
-LINESTRING(4.259 55.858,4.895 55.931,5.533 56.001,6.174 56.067,6.817 56.13,7.462 56.19,8.109 56.246,8.757 56.299,9.408 56.349,10.06 56.395,10.713 56.437,11.368 56.477,12.025 56.513,12.682 56.545,13.341 56.574,14 56.599,14.661 56.621,15.322 56.639,15.983 56.654,16.645 56.666,17.308 56.674,17.97 56.678,18.633 56.679,19.296 56.676,19.958 56.67,20.62 56.66,21.282 56.647,21.943 56.63,22.604 56.61,23.264 56.586,23.923 56.559,24.581 56.528,25.238 56.494,25.893 56.456,26.548 56.415,27.2 56.371,27.852 56.323,28.501 56.271,29.149 56.217,29.795 56.159,30.439 56.097,31.08 56.032,31.72 55.964,32.357 55.893,32.992 55.818,33.624 55.74,34.254 55.659,34.881 55.574,35.505 55.487,36.127 55.396,36.745 55.302,37.361 55.205,37.974 55.105,38.583 55.002,39.189 54.896,39.792 54.787,40.392 54.674,40.989 54.559,41.582 54.441,42.171 54.32,42.758 54.197,43.34 54.07,43.919 53.941,44.494 53.809,45.066 53.674,45.634 53.536,46.199 53.396,46.759 53.253,47.316 53.107,47.869 52.959,48.418 52.809,48.964 52.655,49.505 52.5,50.043 52.342,50.577 52.181,51.107 52.018,51.633 51.853,52.155 51.685,52.673 51.515,53.188 51.343,53.698 51.169,54.205 50.992,54.708 50.813,55.207 50.632,55.702 50.449,56.193 50.264,56.681 50.077,57.164 49.888,57.644 49.697,58.12 49.503,58.593 49.308,59.061 49.111,59.526 48.912,59.987 48.712,60.445 48.509,60.899 48.305,61.349 48.098,61.795 47.89,62.238 47.681,62.678 47.47,63.114 47.257,63.546 47.042,63.975 46.826,64.4 46.608,64.822 46.388,65.241 46.168,65.656 45.945,66.068 45.721,66.477 45.496,66.882 45.269,67.284 45.041,67.683 44.811,68.079 44.58,68.472 44.348,68.861 44.114,69.248 43.879,69.631 43.642,70.011 43.405,70.389 43.166,70.763 42.926,71.135 42.684,71.503 42.442,71.869 42.198,72.232 41.953,72.592 41.708,72.95 41.46,73.304 41.212,73.657 40.963,74.006 40.713)
+LINESTRING(4.259 55.858,5.53349240387128 56.0006659105918,6.81698919498694 56.130094578525,8.10870381314147 56.2461317260662,9.40781156033233 56.3486370295466,10.7134527044527 56.4374849223869,12.0247359780093 56.5125653297878,13.3407424424749 56.573784325367,14.660529681225 56.6210647008095,15.9831362768927 56.654346440595,17.307586522649 56.67358709506,18.6328953115992 56.6787620464102,19.9580731443722 56.6698646638042,21.282131192215 56.6469063452276,22.6040863516019 56.6099164455407,23.922966226566 56.5589420917603,25.2378139766594 56.4940478882858,26.5476929715805 56.4153155163602,27.8516911979552 56.3228432335229,29.1489253693643 56.2167452801302,30.4385446972665 56.0971512011646,31.7197342877491 55.9642050924945,32.9917181368037 55.8180647814723,34.2537617048216 55.6589009522625,35.5051740589896 55.4868962265697,36.7453095800251 55.3022442104976,37.9735692370026 55.1051485181267,39.1894014407465 54.8958217820713,40.3923024922398 54.6744846607816,41.5818166476476 54.4413648517294,42.757535825811 54.1966961188706,43.919098987406 53.9407173419567,45.0661912174019 53.673671594382,46.198542544017 53.3958052553427,47.3159265281308 53.1073671611612,48.4181586571351 52.8086077997244,49.5050945765883 52.499778551104,50.5766281918714 52.1811309766006,51.6326896704254 51.8529161576737,52.673243373185 51.5153840855177,53.6982857415906 51.1687831014009,54.7078431641625 50.8133593873253,55.7019698441171 50.4493565060761,56.6807456869812 50.0770149893128,57.6442742246566 49.6965719720156,58.5926805899637 49.3082608713202,59.5261095533829 48.9123111075629,60.4447236315382 48.5089478652008,61.3487012749643 48.0983918911668,62.2382351408597 47.6808593281578,63.1135304548766 47.2565615803358,63.9748034645285 46.8257052089336,64.822279985501 46.3884918552974,65.6561940410346 45.9451181889661,66.476786593589 45.4957758784676,67.284304367196 45.0406515826125,68.0789987582454 44.5799269601738,68.8611248319107 44.1137786959568,69.6309404010034 43.6423785413868,70.388705183725 43.1658933678633,71.1346800365587 42.6844852312539,71.8691262583921 42.1983114460249,72.5923049618788 41.7075246676227,73.3044765080245 41.2122729818388,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.89507305967085 55.930977446384,5.53349240387128 56.0006659105918,6.17416348361598 56.0670448594645,6.81698919498694 56.130094578525,7.46186995983655 56.1897961993418,8.10870381314147 56.2461317260662,8.75738649688733 56.2990840610623,9.40781156033233 56.3486370295466,10.0598704664666 56.3947754031591,10.7134527044527 56.4374849223869,11.3684459078018 56.4767523177655,12.0247359780093 56.5125653297878,12.6822072133468 56.5449127274491,13.3407424424749 56.573784325367,14.0002231625192 56.5991709994144,14.660529681225 56.6210647008095,15.3215412627822 56.6394584686143,15.9831362768927 56.654346440595,16.6451923506331 56.6657238624055,17.307586522649 56.67358709506,17.9701953992046 56.677933620668,18.6328953115992 56.6787620464102,19.2955624744544 56.6760721067401,19.9580731443722 56.6698646638042,20.6203037784591 56.6601417060788,21.282131192215 56.6469063452276,21.943432716288 56.6301628111935,22.6040863516019 56.6099164455407,23.2639709223762 56.5861736930735,23.922966226566 56.5589420917603,24.5809531832687 56.5282302610022,25.2378139766594 56.4940478882858,25.8934321960358 56.4564057142701,26.5476929715805 56.4153155163602,27.2004831054654 56.3707900908252,27.8516911979552 56.3228432335229,28.5012077681911 56.2714897192993,29.1489253693643 56.2167452801302,29.7947386980206 56.1586265820819,30.4385446972665 56.0971512011646,31.0802426536785 56.0323375981587,31.7197342877491 55.9642050924945,32.3569238377352 55.8927738352675,32.9917181368037 55.8180647814723,33.6240266834038 55.74009966154,34.2537617048216 55.6589009522625,34.8808382139074 55.5744918471876,35.5051740589896 55.4868962265697,36.1266899670207 55.3961386269571,36.7453095800251 55.3022442104976,37.3609594849451 55.2052387340427,37.9735692370026 55.1051485181267,38.5830713767178 55.0020004158976,39.1894014407465 54.8958217820713,39.7924979667135 54.7866404419798,40.3923024922398 54.6744846607816,40.9887595483734 54.5593831128969,41.5818166476476 54.4413648517294,42.1714242670021 54.3204592797319,42.757535825811 54.1966961188706,43.3401076592708 54.0701053815371,43.919098987406 53.9407173419567,44.4944718799548 53.8085625081347,45.0661912174019 53.673671594382,45.6342246484243 53.5360754944551,46.198542544017 53.3958052553427,46.7591179485663 53.2528920517295,47.3159265281308 53.1073671611612,47.8689465161932 52.9592619399335,48.4181586571351 52.8086077997244,48.9635461476859 52.6554361849853,49.5050945765883 52.499778551104,50.0427918627159 52.3416663433486,50.5766281918714 52.1811309766006,51.1065959524853 52.0182038158815,51.6326896704254 51.8529161576737,52.1549059431199 51.6852992120372,52.673243373185 51.5153840855177,53.1877025017413 51.3432017648431,53.6982857415906 51.1687831014009,54.2049973104167 50.9921587964881,54.7078431641625 50.8133593873253,55.2068309307272 50.6324152338211,55.7019698441171 50.4493565060761,56.1932706791714 50.2642131726125,56.6807456869812 50.0770149893128,57.1644085311015 49.8877914890534,57.6442742246566 49.6965719720156,58.1203590684218 49.5033854966561,58.5926805899637 49.3082608713202,59.0612574839055 49.1112266464775,59.5261095533829 48.9123111075629,59.9872576527434 48.7115422684016,60.4447236315382 48.5089478652008,60.8985302798459 48.3045553510865,61.3487012749643 48.0983918911668,61.7952611294973 47.8904843581013,62.2382351408597 47.6808593281578,62.6776493422177 47.4695430777354,63.1135304548766 47.2565615803358,63.5459058421237 47.0419405039633,63.9748034645285 46.8257052089336,64.4002518367009 46.6078807460742,64.822279985501 46.3884918552974,65.2409174096934 46.1675629645276,65.6561940410346 45.9451181889661,66.0681402067793 45.7211813306754,66.476786593589 45.4957758784676,66.8821642128236 45.2689250080781,67.284304367196 45.0406515826125,67.6832386187654 44.8109781532476,68.0789987582454 44.5799269601738,68.4716167756018 44.3475199337644,68.8611248319107 44.1137786959568,69.2475552324516 43.878724561833,69.6309404010034 43.6423785413868,70.0113128553159 43.4047613414637,70.388705183725 43.1658933678633,70.7631500228809 42.925794727592,71.1346800365587 42.6844852312539,71.5033278955199 42.4419843955718,71.8691262583921 42.1983114460249,72.2321077535378 41.953485319597,72.5923049618788 41.7075246676227,72.9497504006463 41.4604478587259,73.3044765080245 41.2122729818388,73.6565156286596 40.963017849297,74.0059 40.7127)

test/CDB_RegenerateTable.sql

@@ -1,292 +0,0 @@
--- Setup
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-SET SESSION AUTHORIZATION postgres;
-\set QUIET off
-
-\echo '## Setup'
-CREATE TABLE testtable (stable integer, c1 integer, c2 integer, c3 integer, c4 integer);
-INSERT INTO testtable(stable,c1,c2,c3,c4) VALUES (1,2,3,4,5), (2,3,4,5,6), (3,4,5,6,7);
-\d+ testtable
-SELECT * FROM testtable ORDER BY stable ASC;
-SELECT 'testtable'::regclass::oid as id INTO temp table original_oid;
-
-CREATE FUNCTION cartodb.CDB_GetTableQueries_TestHelper(tableoid OID, ignore_cartodbfication BOOL DEFAULT false)
-RETURNS text[]
-AS
-$$
-DECLARE
-    queries TEXT[];
-BEGIN
-    -- In older version of PG (pre 11), the syntax when creating triggers was
-    -- CREATE TRIGGER ... EXECUTE PROCEDURE ...
-    -- But in new ones it is
-    -- CREATE TRIGGER ... EXECUTE FUNCTION ...
-    -- To uniformize the tests, we replace it in the output of CDB_RegenerateTable
-
-    EXECUTE FORMAT('
-        SELECT array_agg(REGEXP_REPLACE(a, ''EXECUTE PROCEDURE'', ''EXECUTE FUNCTION''))
-            FROM unnest(cartodb.CDB_GetTableQueries(%L, %L)) a;', tableoid, ignore_cartodbfication) INTO queries;
-    RETURN queries;
-END
-$$
-LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
-
-\echo '## Run cartodb.CDB_RegenerateTable and confirm the data and columns are the same'
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-\d+ testtable
-SELECT * FROM testtable ORDER BY stable ASC;
-
-\echo '## The table oid must have changed since the table itself changed'
-SELECT 'testtable'::regclass::oid as id INTO temp table new_oid;
-SELECT original_oid.id = new_oid.id FROM original_oid, new_oid;
-
-\echo '## Check adding an index'
-CREATE INDEX testtable_stable_idx ON testtable (stable NULLS FIRST) WITH (fillfactor = 80, vacuum_cleanup_index_scale_factor = 0.11);
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-
-\echo '## Check column properties'
-ALTER TABLE testtable ADD UNIQUE (c2);
-ALTER TABLE testtable ALTER COLUMN c3 SET NOT NULL;
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-\echo '## Check triggers'
-CREATE OR REPLACE FUNCTION trigger_example_fn()
-    RETURNS TRIGGER 
-    LANGUAGE PLPGSQL
-AS
-$$
-BEGIN
-	RETURN NEW;
-END;
-$$;
-
-CREATE TRIGGER testtable_trigger_example
-    BEFORE UPDATE
-    ON testtable
-    FOR EACH ROW
-    EXECUTE PROCEDURE trigger_example_fn();
-
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-\echo '## Check Cartodbfycation'
-DROP INDEX testtable_stable_idx;
-DROP TRIGGER testtable_trigger_example ON testtable;
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-SELECT CDB_SetUserQuotaInBytes(0);
-SELECT CDB_CartodbfyTable('testtable'::regclass);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := false);
-SELECT cartodb.CDB_GetTableQueries_TestHelper('testtable'::regclass::oid, ignore_cartodbfication := true);
-
-\d+ testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-SELECT event_object_schema as table_schema,
-       event_object_table as table_name,
-       trigger_schema,
-       trigger_name
-FROM information_schema.triggers
-WHERE event_object_table = 'testtable'
-GROUP BY 1,2,3,4
-ORDER BY table_schema,
-         table_name,
-         trigger_name;
-
-\echo '## Test view / matview dependencies: It will not work but data will be the same'
-CREATE VIEW testview AS SELECT * FROM testtable WHERE stable < 20;
-SELECT * FROM testview ORDER BY stable ASC;
-\d testtable
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-DROP VIEW testview;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-CREATE MATERIALIZED VIEW testmatview AS SELECT * FROM testtable WHERE stable < 20;
-SELECT * FROM testmatview ORDER BY stable ASC;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-DROP MATERIALIZED VIEW testmatview;
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\d testtable
-
-\echo '## Test role access'
-CREATE ROLE cdb_regenerate_tester LOGIN PASSWORD 'cdb_regenerate_pass';
-GRANT CONNECT ON DATABASE contrib_regression TO cdb_regenerate_tester;
-GRANT SELECT ON testtable TO cdb_regenerate_tester;
-
-\c contrib_regression cdb_regenerate_tester
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT * FROM testtable ORDER BY cartodb_id DESC;
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\c contrib_regression cdb_regenerate_tester
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-SELECT * FROM testtable ORDER BY cartodb_id DESC;
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-\echo '## Test calling with read only access (should fail)'
-\c contrib_regression cdb_regenerate_tester
-SELECT cartodb.CDB_RegenerateTable('testtable'::regclass::oid);
-
-\c contrib_regression postgres
-\set QUIET on
-SET client_min_messages TO error;
-\set VERBOSITY terse
-\set QUIET off
-
-\echo '## Test partitioned table'
-CREATE TABLE measurement (
-    city_id         int not null,
-    logdate         date not null,
-    peaktemp        int,
-    unitsales       int
-) PARTITION BY RANGE (logdate);
-
-CREATE TABLE measurement_y2006m02 PARTITION OF measurement
-    FOR VALUES FROM ('2006-02-01') TO ('2006-03-01')
-    PARTITION BY RANGE (peaktemp);
-
-CREATE TABLE measurement_y2006m03 PARTITION OF measurement
-    FOR VALUES FROM ('2006-03-01') TO ('2006-04-01');
-CREATE INDEX ON measurement_y2006m02 (logdate);
-CREATE INDEX ON measurement_y2006m03 (logdate);
-
-\d measurement
-SELECT  c.oid::pg_catalog.regclass,
-        pg_catalog.pg_get_expr(c.relpartbound, c.oid),
-        c.relkind
-FROM pg_catalog.pg_class c,
-     pg_catalog.pg_inherits i
-WHERE c.oid=i.inhrelid AND i.inhparent = 'measurement'::regclass::oid
-ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text;
-\d measurement_y2006m02
-\d measurement_y2006m03
-
-SELECT cartodb.CDB_RegenerateTable('measurement'::regclass::oid);
-SELECT cartodb.CDB_RegenerateTable('measurement_y2006m02'::regclass::oid);
-SELECT  c.oid::pg_catalog.regclass,
-        pg_catalog.pg_get_expr(c.relpartbound, c.oid),
-        c.relkind
-FROM pg_catalog.pg_class c,
-     pg_catalog.pg_inherits i
-WHERE c.oid=i.inhrelid AND i.inhparent = 'measurement'::regclass::oid
-ORDER BY pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT', c.oid::pg_catalog.regclass::pg_catalog.text;
-\d measurement_y2006m02
-\d measurement_y2006m03
-
-SELECT cartodb.CDB_GetTableQueries_TestHelper('measurement'::regclass::oid, ignore_cartodbfication := false);
-
-\echo '## Test transaction with truncate'
-SET statement_timeout = 1000;
-BEGIN;
-    TRUNCATE TABLE testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-COMMIT;
-
-\echo '## Test transaction with delete'
-BEGIN;
-    DELETE FROM testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-COMMIT;
-
-\echo '## Test transaction with delete + cartodbfy'
-BEGIN;
-    INSERT INTO testtable(stable,c1,c2,c3,c4) VALUES (1,2,3,4,5), (2,3,4,5,6), (3,4,5,6,7);
-    DELETE FROM testtable;
-    SELECT CDB_RegenerateTable('public.testtable'::regclass);
-    SELECT CDB_CartodbfyTable('public'::TEXT, 'public.testtable'::REGCLASS);
-COMMIT;
-
-\echo '## Test replacement in import (drop c3 and c4 columns)'
-CREATE INDEX testtable_c4_idx ON testtable (c4 NULLS FIRST);
-
-\d testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-DO $$
-DECLARE
-    queries TEXT[] :=  CDB_GetTableQueries_TestHelper('testtable'::regclass, true);
-BEGIN
-    DROP TABLE testtable;
-    CREATE TABLE testtable (stable integer, c1 integer, c2 integer);
-    PERFORM CDB_CartodbfyTable('public.testtable');
-    PERFORM CDB_ApplyQueriesSafe(queries);
-END$$;
-
-\d testtable
-SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename = 'testtable' ORDER BY tablename, indexname;
-
-\echo '## teardown'
-
-DROP TABLE measurement CASCADE;
-DROP TABLE testtable CASCADE;
-REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_regenerate_tester;
-DROP ROLE cdb_regenerate_tester;
-DROP FUNCTION cartodb.CDB_GetTableQueries_TestHelper;

test/CDB_RegenerateTable_expect

@@ -1,228 +0,0 @@
-## Setup
-CREATE TABLE
-INSERT 0 3
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-1|2|3|4|5
-2|3|4|5|6
-3|4|5|6|7
-SELECT 1
-CREATE FUNCTION
-## Run cartodb.CDB_RegenerateTable and confirm the data and columns are the same
-
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-1|2|3|4|5
-2|3|4|5|6
-3|4|5|6|7
-## The table oid must have changed since the table itself changed
-SELECT 1
-f
-## Check adding an index
-CREATE INDEX
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-## Check column properties
-ALTER TABLE
-ALTER TABLE
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||not null||plain||
-c4|integer||||plain||
-testtable|testtable_c2_key|CREATE UNIQUE INDEX testtable_c2_key ON public.testtable USING btree (c2)
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||not null||plain||
-c4|integer||||plain||
-testtable|testtable_c2_key|CREATE UNIQUE INDEX testtable_c2_key ON public.testtable USING btree (c2)
-testtable|testtable_stable_idx|CREATE INDEX testtable_stable_idx ON public.testtable USING btree (stable NULLS FIRST) WITH (fillfactor='80', vacuum_cleanup_index_scale_factor='0.11')
-## Check triggers
-CREATE FUNCTION
-CREATE TRIGGER
-public|testtable|public|testtable_trigger_example
-
-public|testtable|public|testtable_trigger_example
-## Check Cartodbfycation
-DROP INDEX
-DROP TRIGGER
-{"ALTER TABLE public.testtable OWNER TO postgres","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_c2_key UNIQUE (c2)"}
-{"ALTER TABLE public.testtable OWNER TO postgres","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_c2_key UNIQUE (c2)"}
-0
-testtable
-{"ALTER TABLE public.testtable OWNER TO postgres","CREATE SEQUENCE public.testtable_cartodb_id_seq    START WITH 1    INCREMENT BY 1    NO MINVALUE    NO MAXVALUE    CACHE 1","ALTER TABLE public.testtable_cartodb_id_seq OWNER TO postgres","ALTER SEQUENCE public.testtable_cartodb_id_seq OWNED BY public.testtable.cartodb_id","ALTER TABLE ONLY public.testtable ALTER COLUMN cartodb_id SET DEFAULT nextval('public.testtable_cartodb_id_seq'::regclass)","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_pkey PRIMARY KEY (cartodb_id)","CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)","CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)","CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_checkquota('0.1', '-1', 'public')","CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb.cdb_checkquota('0.001', '-1', 'public')","CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_tablemetadata_trigger()","CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb._cdb_update_the_geom_webmercator()"}
-{"ALTER TABLE public.testtable OWNER TO postgres"}
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)|plain||
-the_geom|geometry(Geometry,4326)||||main||
-the_geom_webmercator|geometry(Geometry,3857)||||main||
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-public|testtable|public|test_quota
-public|testtable|public|test_quota_per_row
-public|testtable|public|track_updates
-public|testtable|public|update_the_geom_webmercator_trigger
-
-{"ALTER TABLE public.testtable OWNER TO postgres","CREATE SEQUENCE public.testtable_cartodb_id_seq    START WITH 1    INCREMENT BY 1    NO MINVALUE    NO MAXVALUE    CACHE 1","ALTER TABLE public.testtable_cartodb_id_seq OWNER TO postgres","ALTER SEQUENCE public.testtable_cartodb_id_seq OWNED BY public.testtable.cartodb_id","ALTER TABLE ONLY public.testtable ALTER COLUMN cartodb_id SET DEFAULT nextval('public.testtable_cartodb_id_seq'::regclass)","ALTER TABLE ONLY public.testtable    ADD CONSTRAINT testtable_pkey PRIMARY KEY (cartodb_id)","CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)","CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)","CREATE TRIGGER test_quota BEFORE INSERT OR UPDATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_checkquota('0.1', '-1', 'public')","CREATE TRIGGER test_quota_per_row BEFORE INSERT OR UPDATE ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb.cdb_checkquota('0.001', '-1', 'public')","CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON public.testtable FOR EACH STATEMENT EXECUTE FUNCTION cartodb.cdb_tablemetadata_trigger()","CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON public.testtable FOR EACH ROW EXECUTE FUNCTION cartodb._cdb_update_the_geom_webmercator()"}
-{"ALTER TABLE public.testtable OWNER TO postgres"}
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)|plain||
-the_geom|geometry(Geometry,4326)||||main||
-the_geom_webmercator|geometry(Geometry,3857)||||main||
-stable|integer||||plain||
-c1|integer||||plain||
-c2|integer||||plain||
-c3|integer||||plain||
-c4|integer||||plain||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-public|testtable|public|test_quota
-public|testtable|public|test_quota_per_row
-public|testtable|public|track_updates
-public|testtable|public|update_the_geom_webmercator_trigger
-## Test view / matview dependencies: It will not work but data will be the same
-CREATE VIEW
-1|||1|2|3|4|5
-2|||2|3|4|5|6
-3|||3|4|5|6|7
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-ERROR:  cannot drop table testtable because other objects depend on it
-DROP VIEW
-
-SELECT 3
-1|||1|2|3|4|5
-2|||2|3|4|5|6
-3|||3|4|5|6|7
-ERROR:  cannot drop table testtable because other objects depend on it
-DROP MATERIALIZED VIEW
-
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-## Test role access
-CREATE ROLE
-GRANT
-GRANT
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-3|||3|4|5|6|7
-2|||2|3|4|5|6
-1|||1|2|3|4|5
-You are now connected to database "contrib_regression" as user "postgres".
-
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-3|||3|4|5|6|7
-2|||2|3|4|5|6
-1|||1|2|3|4|5
-You are now connected to database "contrib_regression" as user "postgres".
-## Test calling with read only access (should fail)
-You are now connected to database "contrib_regression" as user "cdb_regenerate_tester".
-ERROR:  must be owner of table testtable
-You are now connected to database "contrib_regression" as user "postgres".
-## Test partitioned table
-CREATE TABLE
-CREATE TABLE
-CREATE TABLE
-CREATE INDEX
-CREATE INDEX
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-measurement_y2006m02|FOR VALUES FROM ('02-01-2006') TO ('03-01-2006')|p
-measurement_y2006m03|FOR VALUES FROM ('03-01-2006') TO ('04-01-2006')|r
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-ERROR:  CDB_RegenerateTable does not support the parent of partitioned tables
-
-measurement_y2006m02|FOR VALUES FROM ('02-01-2006') TO ('03-01-2006')|p
-measurement_y2006m03|FOR VALUES FROM ('03-01-2006') TO ('04-01-2006')|r
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-city_id|integer||not null|
-logdate|date||not null|
-peaktemp|integer|||
-unitsales|integer|||
-ERROR:  CDB_GetTableQueries does not support the parent of partitioned tables
-## Test transaction with truncate
-SET
-BEGIN
-TRUNCATE TABLE
-ERROR:  plpy.Error: Could not get table properties
-ROLLBACK
-## Test transaction with delete
-BEGIN
-DELETE 3
-
-COMMIT
-## Test transaction with delete + cartodbfy
-BEGIN
-INSERT 0 3
-DELETE 3
-
-testtable
-COMMIT
-## Test replacement in import (drop c3 and c4 columns)
-CREATE INDEX
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-c3|integer|||
-c4|integer|||
-testtable|testtable_c4_idx|CREATE INDEX testtable_c4_idx ON public.testtable USING btree (c4 NULLS FIRST)
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-DO
-cartodb_id|bigint||not null|nextval('testtable_cartodb_id_seq'::regclass)
-the_geom|geometry(Geometry,4326)|||
-the_geom_webmercator|geometry(Geometry,3857)|||
-stable|integer|||
-c1|integer|||
-c2|integer|||
-testtable|testtable_pkey|CREATE UNIQUE INDEX testtable_pkey ON public.testtable USING btree (cartodb_id)
-testtable|testtable_the_geom_idx|CREATE INDEX testtable_the_geom_idx ON public.testtable USING gist (the_geom)
-testtable|testtable_the_geom_webmercator_idx|CREATE INDEX testtable_the_geom_webmercator_idx ON public.testtable USING gist (the_geom_webmercator)
-## teardown
-DROP TABLE
-DROP TABLE
-REVOKE
-DROP ROLE
-DROP FUNCTION