cartodb4/cartodb-postgresql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #107 from CartoDB/new_cartodbfy_rtorre

Browse Source 
Do not create timestamp columns/triggers on cartodbfy
This commit is contained in:
Rafa de la Torre 2015-08-12 18:34:33 +02:00
commit c00d607ee2
6 changed files with 4 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
expected/test_ddl_triggers.out
View File

@ -36,7 +36,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select 1 as i INTO c.t3;
NOTICE: trigger "track_updates" for table "c.t3" does not exist, skipping
NOTICE: trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
NOTICE: trigger "update_updated_at_trigger" for table "c.t3" does not exist, skipping
NOTICE: trigger "test_quota" for table "c.t3" does not exist, skipping
NOTICE: trigger "test_quota_per_row" for table "c.t3" does not exist, skipping
NOTICE: event trigger "cdb_on_relation_create" does not exist, skipping
@ -45,17 +44,6 @@ NOTICE: event trigger "cdb_on_alter_column" does not exist, skipping
NOTICE: event trigger "cdb_on_drop_column" does not exist, skipping
NOTICE: event trigger "cdb_on_add_column" does not exist, skipping
NOTICE: cdb_invalidate_varnish(c.t3) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i
from c.t3;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i
------------+-----+------+----------+----------------------+---
1 | t | t | | | 1
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -72,7 +60,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select 1 as cartodb_id INTO c.t4;
NOTICE: trigger "track_updates" for table "c.t4" does not exist, skipping
NOTICE: trigger "update_the_geom_webmercator_trigger" for table "c.t4" does not exist, skipping
NOTICE: trigger "update_updated_at_trigger" for table "c.t4" does not exist, skipping
NOTICE: trigger "test_quota" for table "c.t4" does not exist, skipping
NOTICE: trigger "test_quota_per_row" for table "c.t4" does not exist, skipping
NOTICE: Column cartodb_id already exists
@ -84,16 +71,6 @@ NOTICE: event trigger "cdb_on_alter_column" does not exist, skipping
NOTICE: event trigger "cdb_on_drop_column" does not exist, skipping
NOTICE: event trigger "cdb_on_add_column" does not exist, skipping
NOTICE: cdb_invalidate_varnish(c.t4) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator
from c.t4;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator
------------+-----+------+----------+----------------------
1 | t | t | |
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -116,8 +93,6 @@ select pg_sleep(.1);
alter table c.t3 rename column the_geom_webmercator to webmerc;
NOTICE: Column cartodb_id already exists
NOTICE: Column created_at already exists
NOTICE: Column updated_at already exists
NOTICE: Column the_geom already exists
NOTICE: event trigger "cdb_on_relation_create" does not exist, skipping
NOTICE: event trigger "cdb_on_relation_drop" does not exist, skipping
@ -125,17 +100,6 @@ NOTICE: event trigger "cdb_on_alter_column" does not exist, skipping
NOTICE: event trigger "cdb_on_drop_column" does not exist, skipping
NOTICE: event trigger "cdb_on_add_column" does not exist, skipping
NOTICE: cdb_invalidate_varnish(c.t3) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc
from c.t3;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc
------------+-----+------+----------+----------------------+---+---------
1 | t | t | | | 1 |
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -155,8 +119,6 @@ select pg_sleep(.1);
alter table c.t3 rename column the_geom_webmercator to webmerc2;
NOTICE: Column cartodb_id already exists
NOTICE: Column created_at already exists
NOTICE: Column updated_at already exists
NOTICE: Column the_geom already exists
NOTICE: event trigger "cdb_on_relation_create" does not exist, skipping
NOTICE: event trigger "cdb_on_relation_drop" does not exist, skipping
@ -164,17 +126,6 @@ NOTICE: event trigger "cdb_on_alter_column" does not exist, skipping
NOTICE: event trigger "cdb_on_drop_column" does not exist, skipping
NOTICE: event trigger "cdb_on_add_column" does not exist, skipping
NOTICE: cdb_invalidate_varnish(c.t3) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2
from c.t3;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2
------------+-----+------+----------+----------------------+---+---------+----------
1 | t | t | | | 1 | |
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -197,8 +148,6 @@ select pg_sleep(.1);
alter table c.t3 drop column the_geom_webmercator;
NOTICE: Column cartodb_id already exists
NOTICE: Column created_at already exists
NOTICE: Column updated_at already exists
NOTICE: Column the_geom already exists
NOTICE: event trigger "cdb_on_relation_create" does not exist, skipping
NOTICE: event trigger "cdb_on_relation_drop" does not exist, skipping
@ -206,17 +155,6 @@ NOTICE: event trigger "cdb_on_alter_column" does not exist, skipping
NOTICE: event trigger "cdb_on_drop_column" does not exist, skipping
NOTICE: event trigger "cdb_on_add_column" does not exist, skipping
NOTICE: cdb_invalidate_varnish(c.t3) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2
from c.t3;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2
------------+-----+------+----------+----------------------+---+---------+----------
1 | t | t | | | 1 | |
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -239,17 +177,6 @@ select pg_sleep(.1);
alter table c.t3 add column id2 int;
NOTICE: cdb_invalidate_varnish(c.t3) called
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2, id2
from c.t3;
cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 | id2
------------+-----+------+----------+----------------------+---+---------+----------+-----
1 | t | t | | | 1 | | |
(1 row)
RESET SESSION AUTHORIZATION;
select
tabname::text,

143
scripts-available/CDB_CartodbfyTable.sql
View File

@ -43,10 +43,6 @@ BEGIN
sql := 'DROP TRIGGER IF EXISTS update_the_geom_webmercator_trigger ON ' || reloid::text;
EXECUTE sql;
-- "update_updated_at"
sql := 'DROP TRIGGER IF EXISTS update_updated_at_trigger ON ' || reloid::text;
EXECUTE sql;
-- "test_quota" and "test_quota_per_row"
sql := 'DROP TRIGGER IF EXISTS test_quota ON ' || reloid::text;
EXECUTE sql;
@ -205,118 +201,6 @@ END;
$$ LANGUAGE PLPGSQL;
-- 4) created_at and updated_at creation & validation or renaming if invalid
CREATE OR REPLACE FUNCTION _CDB_create_timestamp_columns(reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
rec RECORD;
rec2 RECORD;
had_column BOOLEAN;
i INTEGER;
new_name TEXT;
BEGIN
FOR rec IN SELECT * FROM ( VALUES ('created_at'), ('updated_at') ) t(cname)
LOOP --{
new_name := null;
<< column_setup >>
LOOP --{
had_column := FALSE;
BEGIN
sql := 'ALTER TABLE ' || reloid::text || ' ADD ' || rec.cname
|| ' TIMESTAMPTZ NOT NULL DEFAULT now()';
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
EXIT column_setup;
EXCEPTION
WHEN duplicate_column THEN
RAISE NOTICE 'Column % already exists', rec.cname;
had_column := TRUE;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
IF had_column THEN
-- Check data type is a TIMESTAMP WITH TIMEZONE
SELECT t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
INTO STRICT rec2;
IF rec2.oid NOT IN (1184) THEN -- timestamptz {
RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.cname, rec2.typname;
ELSE -- }{
-- Ensure data type is a TIMESTAMP WITH TIMEZONE
sql := 'ALTER TABLE ' || reloid::text
|| ' ALTER ' || rec.cname
|| ' SET NOT NULL,'
|| ' ALTER ' || rec.cname
|| ' SET DEFAULT now()';
BEGIN
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
EXIT column_setup;
EXCEPTION
WHEN not_null_violation THEN -- failed not-null
RAISE NOTICE '%, renaming', SQLERRM;
WHEN cannot_coerce THEN -- failed cast
RAISE NOTICE '%, renaming', SQLERRM;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
END IF; -- }
-- invalid column, need rename and re-create it
i := 0;
<< rename_column >>
LOOP --{
new_name := '_' || rec.cname || i;
BEGIN
sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN ' || rec.cname || ' TO ' || new_name;
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
EXCEPTION
WHEN duplicate_column THEN
i := i+1;
CONTINUE rename_column;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (renaming %): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
END;
EXIT rename_column;
END LOOP; --}
CONTINUE column_setup;
END IF;
END LOOP; -- }
-- Try to copy data from new name if possible
IF new_name IS NOT NULL THEN -- {
RAISE NOTICE 'Trying to recover data from % coumn', new_name;
BEGIN
-- Copy existing values to new field
-- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
-- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
|| ' TYPE TIMESTAMPTZ USING '
|| new_name || '::timestamptz';
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
-- Drop old column (all went find if we got here)
sql := 'ALTER TABLE ' || reloid::text || ' DROP ' || new_name;
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
EXCEPTION
WHEN others THEN
RAISE NOTICE 'Could not initialize % with existing values: % (%)', rec.cname, SQLERRM, SQLSTATE;
END;
END IF; -- }
END LOOP; -- }
END;
$$ LANGUAGE PLPGSQL;
-- 5) the_geom and the_geom_webmercator creation & validation or renaming if invalid
CREATE OR REPLACE FUNCTION _CDB_create_the_geom_columns(reloid REGCLASS)
@ -487,13 +371,6 @@ BEGIN
|| ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()';
EXECUTE sql;
-- "update_updated_at"
-- TODO: why _before_ and not after ?
sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
|| reloid::text
|| ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
EXECUTE sql;
-- "test_quota" and "test_quota_per_row"
sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
@ -526,13 +403,6 @@ BEGIN
|| ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
EXECUTE sql;
-- "update_updated_at"
-- TODO: why _before_ and not after ?
sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
|| reloid::text
|| ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
EXECUTE sql;
-- "test_quota" and "test_quota_per_row"
sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
@ -563,14 +433,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at := now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql VOLATILE;
-- Auxiliary function
CREATE OR REPLACE FUNCTION cartodb._CDB_is_raster_table(schema_name TEXT, reloid REGCLASS)
@ -624,7 +486,6 @@ BEGIN
-- Ensure required fields exist
PERFORM cartodb._CDB_create_cartodb_id_column(reloid);
PERFORM cartodb._CDB_create_timestamp_columns(reloid);
SELECT cartodb._CDB_is_raster_table(schema_name, reloid) INTO is_raster;
IF is_raster THEN
@ -1493,7 +1354,3 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';

40
sql/test_ddl_triggers.sql
View File

@ -21,13 +21,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select 1 as i INTO c.t3;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i
from c.t3;
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -38,11 +31,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-- Table with cartodb_id field, see
-- http://github.com/CartoDB/cartodb-postgresql/issues/32
select 1 as cartodb_id INTO c.t4;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator
from c.t4;
RESET SESSION AUTHORIZATION;
select
@ -58,13 +46,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select pg_sleep(.1);
alter table c.t3 rename column the_geom_webmercator to webmerc;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc
from c.t3;
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -75,13 +56,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select pg_sleep(.1);
alter table c.t3 rename column the_geom_webmercator to webmerc2;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2
from c.t3;
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -95,13 +69,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select pg_sleep(.1);
alter table c.t3 drop column the_geom_webmercator;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2
from c.t3;
RESET SESSION AUTHORIZATION;
select
tabname::text,
@ -115,13 +82,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
select pg_sleep(.1);
alter table c.t3 add column id2 int;
select
cartodb_id, created_at=updated_at as "c=u",
NOW() - updated_at < '1 secs' as "u<1s",
the_geom, the_geom_webmercator,
i, webmerc, webmerc2, id2
from c.t3;
RESET SESSION AUTHORIZATION;
select
tabname::text,

44
test/CDB_CartodbfyTableTest.sql
View File

@ -34,20 +34,10 @@ BEGIN
sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
EXECUTE sql INTO STRICT id;
sql := 'SELECT created_at,updated_at,the_geom_webmercator FROM '
sql := 'SELECT the_geom_webmercator FROM '
|| tabname::text || ' WHERE cartodb_id = ' || id;
EXECUTE sql INTO STRICT rec;
-- Check created_at and updated_at at creation time
lag = rec.created_at - now();
IF lag > '1 second' THEN
RAISE EXCEPTION 'created_at not defaulting to now() after insert [ valued % ago ]', lag;
END IF;
lag = rec.updated_at - now();
IF lag > '1 second' THEN
RAISE EXCEPTION 'updated_at not defaulting to now() after insert [ valued % ago ]', lag;
END IF;
-- Check the_geom_webmercator trigger
IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
@ -115,10 +105,10 @@ BEGIN
-- Check null constraint on cartodb_id, created_at, updated_at
SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
( 'cartodb_id', 'created_at', 'updated_at' )
( 'cartodb_id' )
AND NOT a.attnotnull INTO strict tmp;
IF tmp > 0 THEN
RAISE EXCEPTION 'cartodb_id or created_at or updated_at are missing not-null constraint';
RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
END IF;
-- Cleanup
@ -170,18 +160,6 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
DROP TABLE t;
-- table with existing updated_at and created_at fields ot type text
CREATE TABLE t AS SELECT NOW()::text as created_at,
NOW()::text as updated_at,
NOW() as reftime;
SELECT CDB_CartodbfyTableCheck('t', 'text timestamps');
SELECT extract(secs from reftime-created_at),
extract(secs from reftime-updated_at) FROM t;
CREATE VIEW v AS SELECT * FROM t;
SELECT CDB_CartodbfyTableCheck('t', 'cartodbfied with view');
DROP VIEW v;
DROP TABLE t;
-- table with existing cartodb_id field of type text
CREATE TABLE t AS SELECT 10::text as cartodb_id;
SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
@ -208,22 +186,6 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
DROP TABLE t;
-- table with existing the_geom and created_at and containing null values
-- Really, a test for surviving an longstanding PostgreSQL bug:
-- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
CREATE TABLE t (
the_geom geometry(Geometry,4326),
created_at timestamptz,
updated_at timestamptz
);
COPY t (the_geom, created_at, updated_at) FROM stdin;
0106000020E610000001000000010300000001000000050000009EB8244146435BC017B65E062AD343409EB8244146435BC0F51AF6E2708044400B99891683765AC0F51AF6E2708044400B99891683765AC017B65E062AD343409EB8244146435BC017B65E062AD34340 2012-06-06 21:59:08 2013-06-10 20:17:20
0106000020E61000000100000001030000000100000005000000DA7763431A1A5CC0FBCEE869313C3A40DA7763431A1A5CC09C1B8F55BC494440F9F4A9C7993356C09C1B8F55BC494440F9F4A9C7993356C0FBCEE869313C3A40DA7763431A1A5CC0FBCEE869313C3A40 2012-06-06 21:59:08 2013-06-10 20:17:20
\N \N \N
\.
SELECT CDB_CartodbfyTableCheck('t', 'null geom and timestamp values');
DROP TABLE t;
-- TODO: table with existing custom-triggered the_geom
DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

10
test/CDB_CartodbfyTableTest_expect
View File

@ -28,13 +28,6 @@ trigger-protected the_geom cartodbfied fine
extent|BOX(1 1,2 2)
DROP TABLE
SELECT 1
text timestamps cartodbfied fine
0|0
CREATE VIEW
cartodbfied with view cartodbfied fine
DROP VIEW
DROP TABLE
SELECT 1
text cartodb_id cartodbfied fine
5
DROP TABLE
@ -50,8 +43,5 @@ CREATE TABLE
cartodb_id serial primary key cartodbfied fine
t_pkey|cartodb_id
DROP TABLE
CREATE TABLE
null geom and timestamp values cartodbfied fine
DROP TABLE
DROP FUNCTION
DROP FUNCTION

2
test/CDB_QuotaTest_expect
View File

@ -9,7 +9,7 @@ INSERT 0 2048
INSERT 0 2048
INSERT 0 2048
2
ERROR: Quota exceeded by 567.998046875KB
ERROR: Quota exceeded by 519.998046875KB
0
INSERT 0 1
DROP TABLE