Merge pull request #44 from CartoDB/CDB-3497

Cdb 3497
master 0.3.3
Kartones 10 years ago
commit b75b492490

@ -4,6 +4,7 @@ addons:
postgresql: 9.3
before_install:
- sudo apt-get update
#- sudo apt-get install -q postgresql-9.3-postgis-2.1
- sudo apt-get update
- sudo apt-get install -q postgresql-server-dev-9.3

@ -1,7 +1,7 @@
# cartodb/Makefile
EXTENSION = cartodb
EXTVERSION = 0.3.2
EXTVERSION = 0.3.3
SED = sed
@ -22,6 +22,7 @@ UPGRADABLE = \
0.3.0 \
0.3.0dev \
0.3.1 \
0.3.2 \
$(EXTVERSION)dev \
$(EXTVERSION)next \
$(END)

@ -1,3 +1,7 @@
0.3.3 (2014-07-xx)
------------------
* Splitting of CartodbfyTable method in subfunctions to be able to call in fragments and evade timeouts on hot zones
0.3.2 (2014-07-28)
------------------
* Make 0.3.0dev version upgradeable

@ -6,54 +6,35 @@
-- (user.rebuild_quota_trigger, called by rake task
-- cartodb:db:update_test_quota_trigger)
-- Update the_geom_webmercator
CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
RETURNS trigger AS $$
BEGIN
NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
RETURN NEW;
END;
$$ LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at := now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql VOLATILE;
-- Ensure a table is a "cartodb" table
-- See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table
CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(schema_name TEXT, reloid REGCLASS)
-- 1) Required checks before running cartodbfication
-- Either will pass silenty or raise an exception
CREATE OR REPLACE FUNCTION _CDB_check_prerequisites(schema_name TEXT, reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
rec RECORD;
rec2 RECORD;
tabinfo RECORD;
had_column BOOLEAN;
i INTEGER;
new_name TEXT;
quota_in_bytes INT8;
exists_geom_cols BOOLEAN[];
BEGIN
IF cartodb.schema_exists(schema_name) = false THEN
RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
END IF;
-- TODO: Check that user quota is set ?
BEGIN
-- Content will be discarded
EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name::text) INTO sql;
EXCEPTION WHEN undefined_function THEN
RAISE EXCEPTION 'Please set user quota before cartodbfying tables.';
END;
END;
$$ LANGUAGE PLPGSQL;
-- Drop cartodb triggers (might prevent changing columns)
-- 2) Drop cartodb triggers (might prevent changing columns)
CREATE OR REPLACE FUNCTION _CDB_drop_triggers(reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
BEGIN
-- "track_updates"
sql := 'DROP TRIGGER IF EXISTS track_updates ON ' || reloid::text;
EXECUTE sql;
@ -71,10 +52,22 @@ BEGIN
EXECUTE sql;
sql := 'DROP TRIGGER IF EXISTS test_quota_per_row ON ' || reloid::text;
EXECUTE sql;
END;
$$ LANGUAGE PLPGSQL;
-- Ensure required fields exist
-- We need a cartodb_id column
-- 3) Cartodb_id creation & validation or renaming if invalid
CREATE OR REPLACE FUNCTION _CDB_create_cartodb_id_column(reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
rec RECORD;
rec2 RECORD;
had_column BOOLEAN;
i INTEGER;
new_name TEXT;
BEGIN
<< cartodb_id_setup >>
LOOP --{
had_column := FALSE;
@ -88,12 +81,10 @@ BEGIN
RAISE NOTICE 'Column cartodb_id already exists';
had_column := TRUE;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
reloid, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
END;
IF had_column THEN
SELECT pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id')
AS seq INTO rec2;
@ -101,9 +92,9 @@ BEGIN
SELECT
pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') as seq,
t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
AND a.attname = 'cartodb_id'
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = 'cartodb_id'
INTO STRICT rec;
-- 20=int2, 21=int4, 23=int8
IF rec.oid NOT IN (20,21,23) THEN -- {
RAISE NOTICE 'Existing cartodb_id field is of invalid type % (need int2, int4 or int8), renaming', rec.typname;
@ -128,12 +119,10 @@ BEGIN
WHEN unique_violation OR not_null_violation THEN
RAISE NOTICE '%, renaming', SQLERRM;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
reloid, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
END;
END IF; -- }
-- invalid column, need rename and re-create it
i := 0;
<< rename_column >>
@ -148,13 +137,11 @@ BEGIN
i := i+1;
CONTINUE rename_column;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)',
reloid, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
END;
EXIT rename_column;
END LOOP; --}
CONTINUE cartodb_id_setup;
END IF;
END LOOP; -- }
@ -163,8 +150,7 @@ BEGIN
RAISE NOTICE 'Trying to recover data from % column', new_name;
BEGIN
-- Copy existing values to new field
-- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-- is also known to be faster for tables with many rows
-- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
-- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
sql := 'ALTER TABLE ' || reloid::text
|| ' ALTER cartodb_id TYPE int USING '
@ -187,7 +173,7 @@ BEGIN
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
-- Drop old column (all went find if we got here)
-- Drop old column (all went fine if we got here)
sql := 'ALTER TABLE ' || reloid::text || ' DROP ' || new_name;
RAISE DEBUG 'Running %', sql;
EXECUTE sql;
@ -199,7 +185,23 @@ BEGIN
END;
END IF;
-- We need created_at and updated_at
END;
$$ LANGUAGE PLPGSQL;
-- 4) created_at and updated_at creation & validation or renaming if invalid
CREATE OR REPLACE FUNCTION _CDB_create_timestamp_columns(reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
rec RECORD;
rec2 RECORD;
had_column BOOLEAN;
i INTEGER;
new_name TEXT;
BEGIN
FOR rec IN SELECT * FROM ( VALUES ('created_at'), ('updated_at') ) t(cname)
LOOP --{
new_name := null;
@ -217,20 +219,16 @@ BEGIN
RAISE NOTICE 'Column % already exists', rec.cname;
had_column := TRUE;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
IF had_column THEN
-- Check data type is a TIMESTAMP WITH TIMEZONE
SELECT t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
WHERE a.atttypid = t.oid AND a.attrelid = reloid
AND NOT a.attisdropped AND a.attname = rec.cname
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
INTO STRICT rec2;
IF rec2.oid NOT IN (1184) THEN -- timestamptz {
RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.
cname, rec2.typname;
RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.cname, rec2.typname;
ELSE -- }{
-- Ensure data type is a TIMESTAMP WITH TIMEZONE
sql := 'ALTER TABLE ' || reloid::text
@ -248,8 +246,7 @@ cname, rec2.typname;
WHEN cannot_coerce THEN -- failed cast
RAISE NOTICE '%, renaming', SQLERRM;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
END IF; -- }
@ -273,7 +270,6 @@ cname, rec2.typname;
EXIT rename_column;
END LOOP; --}
CONTINUE column_setup;
END IF;
END LOOP; -- }
@ -282,8 +278,7 @@ cname, rec2.typname;
RAISE NOTICE 'Trying to recover data from % coumn', new_name;
BEGIN
-- Copy existing values to new field
-- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-- is also known to be faster for tables with many rows
-- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
-- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
|| ' TYPE TIMESTAMPTZ USING '
@ -298,13 +293,28 @@ cname, rec2.typname;
EXCEPTION
WHEN others THEN
RAISE NOTICE 'Could not initialize % with existing values: % (%)',
rec.cname, SQLERRM, SQLSTATE;
RAISE NOTICE 'Could not initialize % with existing values: % (%)', rec.cname, SQLERRM, SQLSTATE;
END;
END IF; -- }
END LOOP; -- }
END;
$$ LANGUAGE PLPGSQL;
-- 5) the_geom and the_geom_webmercator creation & validation or renaming if invalid
CREATE OR REPLACE FUNCTION _CDB_create_the_geom_columns(reloid REGCLASS)
RETURNS BOOLEAN[]
AS $$
DECLARE
sql TEXT;
rec RECORD;
rec2 RECORD;
had_column BOOLEAN;
i INTEGER;
new_name TEXT;
exists_geom_cols BOOLEAN[];
BEGIN
-- We need the_geom and the_geom_webmercator
FOR rec IN SELECT * FROM ( VALUES ('the_geom',4326), ('the_geom_webmercator',3857) ) t(cname,csrid) LOOP --{
<< column_setup >> LOOP --{
@ -323,20 +333,17 @@ cname, rec2.typname;
exists_geom_cols := array_append(exists_geom_cols, true);
RAISE NOTICE 'Column % already exists', rec.cname;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
<< column_fixup >>
LOOP --{
-- Check data type is a GEOMETRY
SELECT t.typname, t.oid, a.attnotnull,
postgis_typmod_srid(a.atttypmod) as srid,
postgis_typmod_type(a.atttypmod) as gtype
FROM pg_type t, pg_attribute a
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
AND a.attname = rec.cname
WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
INTO STRICT rec2;
IF rec2.typname NOT IN ('geometry') THEN -- {
@ -376,14 +383,12 @@ cname, rec2.typname;
EXECUTE sql;
EXCEPTION
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (% index): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (% index): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
END IF; -- }
-- if we reached this line, all went good
EXIT column_setup;
END LOOP; -- } column_fixup
-- invalid column, need rename and re-create it
@ -400,37 +405,59 @@ cname, rec2.typname;
i := i+1;
CONTINUE rename_column;
WHEN others THEN
RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)',
reloid, rec.cname, SQLERRM, SQLSTATE;
RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
END;
EXIT rename_column;
END LOOP; --}
CONTINUE column_setup;
END LOOP; -- } column_setup
END LOOP; -- } on expected geometry columns
-- Initialize the_geom with values from the_geom_webmercator
-- do this only if the_geom_webmercator was found (not created)
-- _and_ the_geom was NOT found.
IF exists_geom_cols[2] AND NOT exists_geom_cols[1] THEN
RETURN exists_geom_cols;
END;
$$ LANGUAGE PLPGSQL;
-- 6) Initialize the_geom with values from the_geom_webmercator
-- do this only if the_geom_webmercator was found (not created) and the_geom was NOT found.
CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_from_the_geom_webmercator(reloid REGCLASS, geom_columns_exist BOOLEAN[])
RETURNS void
AS $$
DECLARE
sql TEXT;
BEGIN
IF geom_columns_exist[2] AND NOT geom_columns_exist[1] THEN
sql := 'UPDATE ' || reloid::text || ' SET the_geom = ST_Transform(the_geom_webmercator, 4326) ';
EXECUTE sql;
END IF;
END;
$$ LANGUAGE PLPGSQL;
-- Initialize the_geom_webmercator with values from the_geom
-- do this only if the_geom was found (not created)
-- _and_ the_geom_webmercator was NOT found.
IF exists_geom_cols[1] AND NOT exists_geom_cols[2] THEN
-- 7) Initialize the_geom_webmercator with values from the_geom
-- do this only if the_geom was found (not created) and the_geom_webmercator was NOT found.
CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_webmercator_from_the_geom(reloid REGCLASS, geom_columns_exist BOOLEAN[])
RETURNS void
AS $$
DECLARE
sql TEXT;
BEGIN
IF geom_columns_exist[1] AND NOT geom_columns_exist[2] THEN
sql := 'UPDATE ' || reloid::text || ' SET the_geom_webmercator = public.CDB_TransformToWebmercator(the_geom) ';
EXECUTE sql;
END IF;
END;
$$ LANGUAGE PLPGSQL;
-- Re-create all triggers
-- 8) Create all triggers
-- NOTE: drop/create has the side-effect of re-enabling disabled triggers
CREATE OR REPLACE FUNCTION _CDB_create_triggers(schema_name TEXT, reloid REGCLASS)
RETURNS void
AS $$
DECLARE
sql TEXT;
BEGIN
-- "track_updates"
sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
|| reloid::text
@ -466,16 +493,62 @@ cname, rec2.typname;
|| schema_name::text
|| ''')';
EXECUTE sql;
END;
$$ LANGUAGE PLPGSQL;
-- Update the_geom_webmercator
CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
RETURNS trigger
AS $$
BEGIN
NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
RETURN NEW;
END;
$$ LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at := now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql VOLATILE;
-- ////////////////////////////////////////////////////
-- Ensure a table is a "cartodb" table
-- See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table
CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(schema_name TEXT, reloid REGCLASS)
RETURNS void
AS $$
DECLARE
exists_geom_cols BOOLEAN[];
BEGIN
PERFORM cartodb._CDB_check_prerequisites(schema_name, reloid);
PERFORM cartodb._CDB_drop_triggers(reloid);
-- Ensure required fields exist
PERFORM cartodb._CDB_create_cartodb_id_column(reloid);
PERFORM cartodb._CDB_create_timestamp_columns(reloid);
SELECT cartodb._CDB_create_the_geom_columns(reloid) INTO exists_geom_cols;
-- Both only populate if proceeds
PERFORM cartodb._CDB_populate_the_geom_from_the_geom_webmercator(reloid, exists_geom_cols);
PERFORM cartodb._CDB_populate_the_geom_webmercator_from_the_geom(reloid, exists_geom_cols);
PERFORM cartodb._CDB_create_triggers(schema_name, reloid);
END;
$$ LANGUAGE PLPGSQL;
CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(reloid REGCLASS)
RETURNS void
AS $$
BEGIN
PERFORM public.CDB_CartodbfyTable('public', reloid);
PERFORM cartodb.CDB_CartodbfyTable('public', reloid);
END;
$$
LANGUAGE PLPGSQL;
$$ LANGUAGE PLPGSQL;

@ -2,6 +2,8 @@ SET
CREATE FUNCTION
SELECT 1
ERROR: Please set user quota before cartodbfying tables.
CONTEXT: SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
PL/pgSQL function cdb_cartodbfytable(text,regclass) line 6 at PERFORM
0
single non-geometrical column cartodbfied fine
DROP TABLE

Loading…
Cancel
Save