From 777e8c6e4cc648f2d66db2d4e2c1a7ba4d6551ea Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 29 Sep 2015 18:03:20 +0200 Subject: [PATCH 01/36] Adds test to check cartodbfy works with null cartodb_id --- test/CDB_CartodbfyTableTest.sql | 40 +++++++++++++++++++++++++++++- test/CDB_CartodbfyTableTest_expect | 23 +++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index b35540b..42a23cf 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -250,7 +250,7 @@ INSERT INTO existing_cartodb_id (cartodb_id, description) VALUES (20, 'b'), (30, 'c'); SELECT CDB_CartodbfyTableCheck('existing_cartodb_id', 'Existing cartodb_id values are respected #138'); -SELECT * from existing_cartodb_id; +SELECT cartodb_id,the_geom,the_geom_webmercator,description,name from existing_cartodb_id; DROP TABLE existing_cartodb_id; -- Table with both the_geom and wkb_geometry @@ -281,6 +281,44 @@ INSERT INTO many_colliding_columns VALUES ( SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141'); DROP TABLE many_colliding_columns; +-- table with null cartodb_id +CREATE TABLE test ( + cartodb_id integer +); +INSERT INTO test VALUES + (1), + (2), + (NULL), + (3); +SELECT CDB_CartodbfyTable('test'); +SELECT cartodb_id, cartodb_id_1 from test; +DROP TABLE test; + +-- table with non unique cartodb_id +CREATE TABLE test ( + cartodb_id integer +); +INSERT INTO test VALUES + (1), + (2), + (2); +SELECT CDB_CartodbfyTable('test'); +SELECT cartodb_id, cartodb_id_1 from test; +DROP TABLE test; + +-- table with non unique and null cartodb_id +CREATE TABLE test ( + cartodb_id integer +); +INSERT INTO test VALUES + (1), + (2), + (NULL), + (2); +SELECT CDB_CartodbfyTable('test'); +SELECT cartodb_id, cartodb_id_1 from test; +DROP TABLE test; + -- TODO: table with existing custom-triggered the_geom diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 6287c3f..ab6c8e9 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -79,5 +79,28 @@ CREATE TABLE INSERT 0 1 Many colliding columns #141 cartodbfied fine DROP TABLE +CREATE TABLE +INSERT 0 4 +test +1|1 +2|2 +3| +4|3 +DROP TABLE +CREATE TABLE +INSERT 0 3 +test +1|1 +2|2 +3|2 +DROP TABLE +CREATE TABLE +INSERT 0 4 +test +1|1 +2|2 +3| +4|2 +DROP TABLE DROP FUNCTION DROP FUNCTION From 3429e93979dedc20ab1570c7beae519eeb74cfb4 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 29 Sep 2015 18:04:00 +0200 Subject: [PATCH 02/36] Makes cartodbfy check for null cartodb_id --- scripts-available/CDB_CartodbfyTable.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 2e44e9a..84a6d51 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -362,7 +362,7 @@ $$ LANGUAGE PLPGSQL; -- As before, this drops all the metadata and geom sync triggers -- -- (2) _CDB_Has_Usable_Primary_ID() --- Returns TRUE if it can find a unique integer primary key named +-- Returns TRUE if it can find a unique and not null integer primary key named -- 'cartodb_id' or can rename an existing key. -- Returns FALSE otherwise. -- @@ -551,7 +551,7 @@ BEGIN RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey); RETURN true; - -- Check and see if the column values are unique, + -- Check and see if the column values are unique and not null, -- if they are, we can use this column... ELSE @@ -559,13 +559,17 @@ BEGIN useable_key := true; BEGIN - sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_unique UNIQUE (%s)', reloid::text, const.pkey, const.pkey); + sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey); RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql; EXECUTE sql; EXCEPTION -- Failed unique check... WHEN unique_violation THEN - RAISE NOTICE 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey); + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey); + useable_key := false; + -- Failed not null check... + WHEN not_null_violation THEN + RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey); useable_key := false; -- Other fatal error WHEN others THEN @@ -574,7 +578,7 @@ BEGIN -- Clean up test constraint IF useable_key THEN - PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_unique', reloid::text, const.pkey)); + PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey)); -- Move non-unique column out of the way ELSE From 5de395a4a8639c6e3a3e3929ef2c5b78db5eb1bd Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 29 Sep 2015 19:01:30 +0200 Subject: [PATCH 03/36] Uses CDB_CartodbfyTableCheck() 148 --- test/CDB_CartodbfyTableTest.sql | 12 ++++++------ test/CDB_CartodbfyTableTest_expect | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 42a23cf..dc4bab7 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -281,7 +281,7 @@ INSERT INTO many_colliding_columns VALUES ( SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141'); DROP TABLE many_colliding_columns; --- table with null cartodb_id +-- Table with null cartodb_id CREATE TABLE test ( cartodb_id integer ); @@ -290,11 +290,11 @@ INSERT INTO test VALUES (2), (NULL), (3); -SELECT CDB_CartodbfyTable('test'); +SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148'); SELECT cartodb_id, cartodb_id_1 from test; DROP TABLE test; --- table with non unique cartodb_id +-- Table with non unique cartodb_id CREATE TABLE test ( cartodb_id integer ); @@ -302,11 +302,11 @@ INSERT INTO test VALUES (1), (2), (2); -SELECT CDB_CartodbfyTable('test'); +SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148'); SELECT cartodb_id, cartodb_id_1 from test; DROP TABLE test; --- table with non unique and null cartodb_id +-- Table with non unique and null cartodb_id CREATE TABLE test ( cartodb_id integer ); @@ -315,7 +315,7 @@ INSERT INTO test VALUES (2), (NULL), (2); -SELECT CDB_CartodbfyTable('test'); +SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148'); SELECT cartodb_id, cartodb_id_1 from test; DROP TABLE test; diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index ab6c8e9..6103dc8 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -81,7 +81,7 @@ Many colliding columns #141 cartodbfied fine DROP TABLE CREATE TABLE INSERT 0 4 -test +Table with null cartodb_id #148 cartodbfied fine 1|1 2|2 3| @@ -89,14 +89,14 @@ test DROP TABLE CREATE TABLE INSERT 0 3 -test +Table with non unique cartodb_id #148 cartodbfied fine 1|1 2|2 3|2 DROP TABLE CREATE TABLE INSERT 0 4 -test +Table with non unique and null cartodb_id #148 cartodbfied fine 1|1 2|2 3| From e4ce12d1a3bcf9abda7130009ca3739c1f750699 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 5 Oct 2015 11:51:13 +0200 Subject: [PATCH 04/36] Changes CDB_DateToNumber() to accept timestamps with time zones 2202 --- scripts-available/CDB_DateToNumber.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_DateToNumber.sql b/scripts-available/CDB_DateToNumber.sql index 1b9a449..84c3395 100644 --- a/scripts-available/CDB_DateToNumber.sql +++ b/scripts-available/CDB_DateToNumber.sql @@ -1,6 +1,6 @@ -- Convert timestamp to double precision -- -CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp) +CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp with time zone) RETURNS double precision AS $$ DECLARE output double precision; BEGIN From 186ed34ee54eb43a36d433078dc6ef8a768e9bb7 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 5 Oct 2015 13:32:03 +0200 Subject: [PATCH 05/36] Overloads CDB_DateToNumber() and adds test --- scripts-available/CDB_DateToNumber.sql | 16 ++++++++++++++++ test/CDB_DateToNumberTest.sql | 2 ++ test/CDB_DateToNumberTest_expect | 2 ++ 3 files changed, 20 insertions(+) create mode 100644 test/CDB_DateToNumberTest.sql create mode 100644 test/CDB_DateToNumberTest_expect diff --git a/scripts-available/CDB_DateToNumber.sql b/scripts-available/CDB_DateToNumber.sql index 84c3395..651d429 100644 --- a/scripts-available/CDB_DateToNumber.sql +++ b/scripts-available/CDB_DateToNumber.sql @@ -1,5 +1,21 @@ -- Convert timestamp to double precision -- +CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp) +RETURNS double precision AS $$ +DECLARE output double precision; +BEGIN + BEGIN + SELECT extract (EPOCH FROM input) INTO output; + EXCEPTION WHEN OTHERS THEN + RETURN NULL; + END; +RETURN output; +END; +$$ +LANGUAGE 'plpgsql' STABLE STRICT; + +-- Convert timestamp with time zone to double precision +-- CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp with time zone) RETURNS double precision AS $$ DECLARE output double precision; diff --git a/test/CDB_DateToNumberTest.sql b/test/CDB_DateToNumberTest.sql new file mode 100644 index 0000000..3821bd1 --- /dev/null +++ b/test/CDB_DateToNumberTest.sql @@ -0,0 +1,2 @@ +SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+00'::timestamp); +SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+00'::timestamp with time zone); diff --git a/test/CDB_DateToNumberTest_expect b/test/CDB_DateToNumberTest_expect new file mode 100644 index 0000000..8445360 --- /dev/null +++ b/test/CDB_DateToNumberTest_expect @@ -0,0 +1,2 @@ +915753600 +915753600 From 6b3b28f01f20bdaf829aacffe9c2260bd745367e Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 5 Oct 2015 14:56:45 +0200 Subject: [PATCH 06/36] Makes CDB_DateToNumber() tests more intuitive 2202 --- test/CDB_DateToNumberTest.sql | 4 ++-- test/CDB_DateToNumberTest_expect | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CDB_DateToNumberTest.sql b/test/CDB_DateToNumberTest.sql index 3821bd1..0c1051c 100644 --- a/test/CDB_DateToNumberTest.sql +++ b/test/CDB_DateToNumberTest.sql @@ -1,2 +1,2 @@ -SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+00'::timestamp); -SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+00'::timestamp with time zone); +SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00'::timestamp); +SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+05'::timestamp with time zone); diff --git a/test/CDB_DateToNumberTest_expect b/test/CDB_DateToNumberTest_expect index 8445360..bd3d3b0 100644 --- a/test/CDB_DateToNumberTest_expect +++ b/test/CDB_DateToNumberTest_expect @@ -1,2 +1,2 @@ 915753600 -915753600 +915735600 From 03636fafc479b588a59591d84a3cf98126978398 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 6 Oct 2015 11:33:56 +0200 Subject: [PATCH 07/36] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 976b4ca..ea934b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ the extension into your test database. During development the cartodb extension version doesn't change with every commit, so testing latest change requires cheating with PostgreSQL -so to enforce re-load of the scripts. To help with cheating, "make install" +as to enforce the scripts to reload. To help with cheating, "make install" also installs migration scripts to go from "V" to "V"next and from "V"next to "V". Example to upgrade a 0.2.0dev version: From 913bfd72f88bf47af5c6fa65db3a41783e8e55a9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 6 Oct 2015 14:45:35 +0200 Subject: [PATCH 08/36] New version 0.11.1 - Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169) - cartodbfy now discards cartodb_id candidates that contain nulls [#148](https://github.com/CartoDB/cartodb-postgresql/issues/148) --- Makefile | 3 ++- NEWS.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0a5b72d..8f08ebe 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.11.0 +EXTVERSION = 0.11.1 SED = sed @@ -51,6 +51,7 @@ UPGRADABLE = \ 0.10.1 \ 0.10.2 \ 0.11.0 \ + 0.11.1 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index 24e604c..e3b2a3b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,11 @@ next (2015-mm-dd) ----------------- +0.11.1 (2015-10-06) +------------------- +* Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169) +* cartodbfy now discards cartodb_id candidates that contain nulls [#148](https://github.com/CartoDB/cartodb-postgresql/issues/148) + 0.11.0 (2015-09-dd) ------------------- * Groups API From 1f63811383866efb73d17eec292e18dfe812f722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Tue, 6 Oct 2015 18:09:37 +0200 Subject: [PATCH 09/36] Fix schema not being specified on pg_get_serial_sequence --- scripts-available/CDB_Groups.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_Groups.sql b/scripts-available/CDB_Groups.sql index 826c210..43ba023 100644 --- a/scripts-available/CDB_Groups.sql +++ b/scripts-available/CDB_Groups.sql @@ -163,7 +163,7 @@ BEGIN group_role := cartodb._CDB_Group_GroupRole(group_name); FOR column_name IN EXECUTE 'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = current_database() AND TABLE_SCHEMA = $1 AND TABLE_NAME = $2 AND COLUMN_DEFAULT LIKE ''nextval%''' USING username, table_name LOOP - EXECUTE 'SELECT PG_GET_SERIAL_SEQUENCE($1, $2)' USING table_name, column_name INTO sequence_name; + EXECUTE format('SELECT PG_GET_SERIAL_SEQUENCE(''%I.%I'', ''%I'')', username, table_name, column_name) INTO sequence_name; IF sequence_name IS NOT NULL THEN IF do_grant THEN -- Here %s is needed since sequence_name has quotes From 37160c7b357b04d96b227d3279f4cddc9c68f7ca Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Sun, 27 Sep 2015 19:22:22 +0200 Subject: [PATCH 10/36] Write invalidation duration in postgresql log --- scripts-available/CDB_TableMetadata.sql | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_TableMetadata.sql b/scripts-available/CDB_TableMetadata.sql index d91f798..7eb22b7 100644 --- a/scripts-available/CDB_TableMetadata.sql +++ b/scripts-available/CDB_TableMetadata.sql @@ -64,6 +64,10 @@ DECLARE tabname TEXT; rec RECORD; found BOOL; + function_start timestamptz; + function_end timestamptz; + function_duration float; + log_error_verbosity text; BEGIN IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN @@ -72,6 +76,8 @@ BEGIN tabname = OLD.tabname; END IF; + EXECUTE 'SELECT clock_timestamp()' INTO function_start; + -- Notify table data update -- This needs a little bit more of research regarding security issues -- see https://github.com/CartoDB/cartodb/pull/241 @@ -105,7 +111,15 @@ BEGIN EXIT; END LOOP; IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF; - + + EXECUTE 'SELECT clock_timestamp()' INTO function_end; + SELECT extract(epoch from (function_end - function_start)) INTO function_duration; + + EXECUTE 'SELECT setting FROM pg_settings where name=''log_error_verbosity''' INTO log_error_verbosity; + SET log_error_verbosity=TERSE; + RAISE LOG 'invalidation_duration: %', function_duration::text; + EXECUTE 'SET log_error_verbosity= ' || log_error_verbosity; + RETURN NULL; END; $$ From d1d5ed6df3ab75e71000f50adc0b235ffef0327b Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 19 Oct 2015 14:09:52 +0200 Subject: [PATCH 11/36] Optimize invalidation time logging --- scripts-available/CDB_TableMetadata.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_TableMetadata.sql b/scripts-available/CDB_TableMetadata.sql index 7eb22b7..858d36b 100644 --- a/scripts-available/CDB_TableMetadata.sql +++ b/scripts-available/CDB_TableMetadata.sql @@ -67,7 +67,7 @@ DECLARE function_start timestamptz; function_end timestamptz; function_duration float; - log_error_verbosity text; + log_error_verbosity_value text; BEGIN IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN @@ -76,7 +76,7 @@ BEGIN tabname = OLD.tabname; END IF; - EXECUTE 'SELECT clock_timestamp()' INTO function_start; + function_start := clock_timestamp(); -- Notify table data update -- This needs a little bit more of research regarding security issues @@ -112,13 +112,12 @@ BEGIN END LOOP; IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF; - EXECUTE 'SELECT clock_timestamp()' INTO function_end; + function_end := clock_timestamp(); SELECT extract(epoch from (function_end - function_start)) INTO function_duration; - - EXECUTE 'SELECT setting FROM pg_settings where name=''log_error_verbosity''' INTO log_error_verbosity; + SELECT setting INTO log_error_verbosity_value FROM pg_settings WHERE name='log_error_verbosity'; SET log_error_verbosity=TERSE; RAISE LOG 'invalidation_duration: %', function_duration::text; - EXECUTE 'SET log_error_verbosity= ' || log_error_verbosity; + PERFORM 'SET log_error_verbosity= ' || log_error_verbosity_value; RETURN NULL; END; From cf2587ca542d12f6af9ce92f8004bb21cafe0dff Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 19 Oct 2015 14:35:10 +0200 Subject: [PATCH 12/36] New version 0.11.2 --- Makefile | 3 ++- NEWS.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8f08ebe..8350b09 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.11.1 +EXTVERSION = 0.11.2 SED = sed @@ -52,6 +52,7 @@ UPGRADABLE = \ 0.10.2 \ 0.11.0 \ 0.11.1 \ + 0.11.2 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index e3b2a3b..42713d2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,11 @@ next (2015-mm-dd) ----------------- +0.11.2 (2015-10-19) +------------------- +* Fix schema not being specified on pg_get_serial_sequence [#170](https://github.com/CartoDB/cartodb-postgresql/pull/170) +* Log invalidation function call duration in seconds [#163](https://github.com/CartoDB/cartodb-postgresql/pull/163) + 0.11.1 (2015-10-06) ------------------- * Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169) From 7c14dd82124617426ce0554f5835dc7743d9abe3 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Wed, 21 Oct 2015 17:33:01 +0200 Subject: [PATCH 13/36] Adds new functions to generate length aware identifiers 172 --- scripts-available/CDB_CartodbfyTable.sql | 114 +++++++++++++++++++++-- 1 file changed, 104 insertions(+), 10 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 84a6d51..e96607d 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -427,8 +427,9 @@ END; $$ LANGUAGE 'plpgsql'; --- Find a unique relation name in the given schema, starting from the --- template given. If the template is already unique, just return it; +-- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe. +-- Find a unique relation name in the given schema, starting from the +-- template given. If the template is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. CREATE OR REPLACE FUNCTION _CDB_Unique_Relation_Name(schemaname TEXT, relationname TEXT) RETURNS TEXT @@ -449,20 +450,114 @@ BEGIN JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.relname = newrelname AND n.nspname = schemaname; - + IF NOT FOUND THEN RETURN newrelname; END IF; - + i := i + 1; newrelname := relationname || '_' || i; - + IF i > 100 THEN PERFORM _CDB_Error('looping too far', '_CDB_Unique_Relation_Name'); END IF; - + END LOOP; - + +END; +$$ LANGUAGE 'plpgsql'; + + +-- UTF8 safe. Find a unique identifier with a given prefix and/or suffix and withing a schema. +CREATE OR REPLACE FUNCTION _CDB_Unique_Identifier(prefix TEXT, schema TEXT, relname TEXT, suffix TEXT) +RETURNS TEXT +AS $$ +DECLARE + rec RECORD; + usedspace INTEGER; + identifier TEXT; + i INTEGER; + originalidentifier TEXT; + maxlen INTEGER; +BEGIN + maxlen := 63; + + usedspace := 3; + usedspace := usedspace + COALESCE(octet_length(prefix), 0); + usedspace := usedspace + COALESCE(octet_length(suffix), 0); + + relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + + IF relname = '' THEN + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); + END IF; + + identifier := COALESCE(prefix, '') || relname || COALESCE(suffix, ''); + + i := 0; + originalidentifier := identifier; + + WHILE i < 100 LOOP + SELECT c.relname, n.nspname + INTO rec + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = identifier + AND n.nspname = schema; + + IF NOT FOUND THEN + RETURN identifier; + END IF; + + identifier := originalidentifier || '_' || i; + i := i + 1; + END LOOP; + + PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier'); +END; +$$ LANGUAGE 'plpgsql'; + + +-- Trims the end of a given string by the given number of octets taking care +-- not to leave characters in half. UTF8 safe. +CREATE OR REPLACE FUNCTION _CDB_Octet_Trim(tostrip TEXT, octets INTEGER) +RETURNS TEXT +AS $$ +DECLARE + expected INTEGER; + examined INTEGER; + tostriplen INTEGER; + charlen INTEGER; + + i INTEGER; + tail TEXT; + + trimmed TEXT; +BEGIN + charlen := bit_length('a'); + tostriplen := char_length(tostrip); + expected := tostriplen * charlen; + examined := bit_length(tostrip); + + IF expected = examined OR octets = 0 THEN + RETURN SUBSTRING(tostrip from 1 for (tostriplen - octets)); + ELSIF octets < 0 THEN + RETURN tostrip; + ELSIF (octets * charlen) > examined THEN + RETURN ''; + END IF; + + i := tostriplen - ((octets - 1) / 2); + LOOP + tail := SUBSTRING(tostrip from i for tostriplen); + + EXIT WHEN octet_length(tail) >= octets OR i <= 0; + + i := i - 1; + END LOOP; + + trimmed := SUBSTRING(tostrip from 1 for (tostriplen - char_length(tail))); + RETURN trimmed; END; $$ LANGUAGE 'plpgsql'; @@ -961,15 +1056,14 @@ BEGIN -- Put the primary key sequence in the right schema -- If the new table is not moving, better ensure the sequence name -- is unique - destseq := relname || '_' || const.pkey || '_seq'; - destseq := _CDB_Unique_Relation_Name(destschema, destseq); + destseq := _CDB_Unique_Identifier(NULL, destschema, relname, '_' || const.pkey || '_seq'); destseq := Format('%I.%I', destschema, destseq); PERFORM _CDB_SQL(Format('CREATE SEQUENCE %s', destseq), '_CDB_Rewrite_Table'); -- Salt a temporary table name if we are re-writing in place -- Note copyname is already escaped and safe to use as identifier IF destschema = relschema THEN - copyname := Format('%I.%I', destschema, Format('%s_%s', destname, salt)); + copyname := Format('%I.%I', destschema, _CDB_Unique_Identifier(NULL, destschema, destname, '_' || salt)); ELSE copyname := Format('%I.%I', destschema, destname); END IF; From 5f46ff10e985be4d2d6d6a7e2af98cd3267efd5c Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Wed, 21 Oct 2015 17:35:13 +0200 Subject: [PATCH 14/36] Doc change 172 --- scripts-available/CDB_CartodbfyTable.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index e96607d..f67f8a8 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -427,8 +427,8 @@ END; $$ LANGUAGE 'plpgsql'; --- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe. --- Find a unique relation name in the given schema, starting from the +-- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe and length +-- aware. Find a unique relation name in the given schema, starting from the -- template given. If the template is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. CREATE OR REPLACE FUNCTION _CDB_Unique_Relation_Name(schemaname TEXT, relationname TEXT) From 5dfe56a6642d4ea7203b4c3a385d8f8a7404b880 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 12:38:00 +0200 Subject: [PATCH 15/36] Moves new CDB_Unique_Identifier and _CDB_Octet_Trim to CDB_Helper.sql file --- Makefile | 1 + scripts-available/CDB_CartodbfyTable.sql | 116 +++-------------------- scripts-available/CDB_Helper.sql | 102 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 104 deletions(-) create mode 100644 scripts-available/CDB_Helper.sql diff --git a/Makefile b/Makefile index 8350b09..b912941 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ CDBSCRIPTS = \ scripts-available/CDB_DDLTriggers.sql \ scripts-available/CDB_ExtensionPost.sql \ scripts-available/CDB_ExtensionUtils.sql \ + scripts-available/CDB_Helper.sql \ $(END) UPGRADABLE = \ diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index f67f8a8..ace030b 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -1,4 +1,5 @@ -- Depends on: +-- * CDB_Helper.sql -- * CDB_ExtensionUtils.sql -- * CDB_TransformToWebmercator.sql -- * CDB_TableMetadata.sql @@ -427,7 +428,7 @@ END; $$ LANGUAGE 'plpgsql'; --- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe and length +-- DEPRECATED: Use cartodb.CDB_Unique_Identifier since it's UTF8 Safe and length -- aware. Find a unique relation name in the given schema, starting from the -- template given. If the template is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. @@ -468,102 +469,9 @@ END; $$ LANGUAGE 'plpgsql'; --- UTF8 safe. Find a unique identifier with a given prefix and/or suffix and withing a schema. -CREATE OR REPLACE FUNCTION _CDB_Unique_Identifier(prefix TEXT, schema TEXT, relname TEXT, suffix TEXT) -RETURNS TEXT -AS $$ -DECLARE - rec RECORD; - usedspace INTEGER; - identifier TEXT; - i INTEGER; - originalidentifier TEXT; - maxlen INTEGER; -BEGIN - maxlen := 63; - - usedspace := 3; - usedspace := usedspace + COALESCE(octet_length(prefix), 0); - usedspace := usedspace + COALESCE(octet_length(suffix), 0); - - relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); - - IF relname = '' THEN - PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); - END IF; - - identifier := COALESCE(prefix, '') || relname || COALESCE(suffix, ''); - - i := 0; - originalidentifier := identifier; - - WHILE i < 100 LOOP - SELECT c.relname, n.nspname - INTO rec - FROM pg_class c - JOIN pg_namespace n ON c.relnamespace = n.oid - WHERE c.relname = identifier - AND n.nspname = schema; - - IF NOT FOUND THEN - RETURN identifier; - END IF; - - identifier := originalidentifier || '_' || i; - i := i + 1; - END LOOP; - - PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier'); -END; -$$ LANGUAGE 'plpgsql'; - - --- Trims the end of a given string by the given number of octets taking care --- not to leave characters in half. UTF8 safe. -CREATE OR REPLACE FUNCTION _CDB_Octet_Trim(tostrip TEXT, octets INTEGER) -RETURNS TEXT -AS $$ -DECLARE - expected INTEGER; - examined INTEGER; - tostriplen INTEGER; - charlen INTEGER; - - i INTEGER; - tail TEXT; - - trimmed TEXT; -BEGIN - charlen := bit_length('a'); - tostriplen := char_length(tostrip); - expected := tostriplen * charlen; - examined := bit_length(tostrip); - - IF expected = examined OR octets = 0 THEN - RETURN SUBSTRING(tostrip from 1 for (tostriplen - octets)); - ELSIF octets < 0 THEN - RETURN tostrip; - ELSIF (octets * charlen) > examined THEN - RETURN ''; - END IF; - - i := tostriplen - ((octets - 1) / 2); - LOOP - tail := SUBSTRING(tostrip from i for tostriplen); - - EXIT WHEN octet_length(tail) >= octets OR i <= 0; - - i := i - 1; - END LOOP; - - trimmed := SUBSTRING(tostrip from 1 for (tostriplen - char_length(tail))); - RETURN trimmed; -END; -$$ LANGUAGE 'plpgsql'; - - --- Find a unique column name in the given relation, starting from the --- column name given. If the column name is already unique, just return it; +-- DEPRECATED: Use cartodb.CDB_Unique_Identifier since it's UTF8 Safe and length +-- aware. Find a unique column name in the given relation, starting from the +-- column name given. If the column name is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. CREATE OR REPLACE FUNCTION _CDB_Unique_Column_Name(reloid REGCLASS, columnname TEXT) RETURNS TEXT @@ -684,7 +592,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, rec.attname, - _CDB_Unique_Column_Name(reloid, const.pkey)), + cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || const.pkey)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -701,7 +609,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', - reloid::text, rec.attname, _CDB_Unique_Column_Name(reloid, const.pkey)), + reloid::text, rec.attname, cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || const.pkey)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -867,7 +775,7 @@ BEGIN WHEN others THEN IF SQLERRM = 'parse error - invalid geometry' THEN text_geom_column := false; - str := _CDB_Unique_Column_Name(reloid, r1.attname); + str := cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || r1.attname); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', @@ -879,7 +787,7 @@ BEGIN -- Just change its name so we can write a new column into that name. ELSE - str := _CDB_Unique_Column_Name(reloid, r1.attname); + str := cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || r1.attname); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', @@ -1056,18 +964,18 @@ BEGIN -- Put the primary key sequence in the right schema -- If the new table is not moving, better ensure the sequence name -- is unique - destseq := _CDB_Unique_Identifier(NULL, destschema, relname, '_' || const.pkey || '_seq'); + destseq := cartodb.CDB_Unique_Identifier(NULL, relname, '_' || const.pkey || '_seq', destschema); destseq := Format('%I.%I', destschema, destseq); PERFORM _CDB_SQL(Format('CREATE SEQUENCE %s', destseq), '_CDB_Rewrite_Table'); -- Salt a temporary table name if we are re-writing in place -- Note copyname is already escaped and safe to use as identifier IF destschema = relschema THEN - copyname := Format('%I.%I', destschema, _CDB_Unique_Identifier(NULL, destschema, destname, '_' || salt)); + copyname := Format('%I.%I', destschema, cartodb.CDB_Unique_Identifier(NULL, destname, '_' || salt), destschema); ELSE copyname := Format('%I.%I', destschema, destname); END IF; - + -- Start building the SQL! sql := Format('CREATE TABLE %s AS SELECT ', copyname); diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql new file mode 100644 index 0000000..013f3ad --- /dev/null +++ b/scripts-available/CDB_Helper.sql @@ -0,0 +1,102 @@ +-- UTF8 safe and lenght aware. Find a unique identifier with a given prefix +-- and/or suffix and withing a schema. +CREATE OR REPLACE FUNCTION cartodb.CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL) +RETURNS TEXT +AS $$ +DECLARE + rec RECORD; + usedspace INTEGER; + ident TEXT; + i INTEGER; + origident TEXT; + maxlen INTEGER; +BEGIN + maxlen := 63; + + usedspace := 3; + usedspace := usedspace + coalesce(octet_length(prefix), 0); + usedspace := usedspace + coalesce(octet_length(suffix), 0); + + relname := CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + + IF relname = '' THEN + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); + END IF; + + ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); + + i := 0; + origident := ident; + + WHILE i < 100 LOOP + IF schema IS NOT NULL THEN + SELECT c.relname, n.nspname + INTO rec + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = ident + AND n.nspname = schema; + ELSE + SELECT c.relname, n.nspname + INTO rec + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = ident; + END IF; + + IF NOT FOUND THEN + RETURN ident; + END IF; + + ident := origident || '_' || i; + i := i + 1; + END LOOP; + + PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier'); +END; +$$ LANGUAGE 'plpgsql'; + + +-- Trims the end of a given string by the given number of octets taking care +-- not to leave characters in half. UTF8 safe. +CREATE OR REPLACE FUNCTION cartodb.CDB_Octet_Trim(tostrip TEXT, octets INTEGER) +RETURNS TEXT +AS $$ +DECLARE + expected INTEGER; + examined INTEGER; + tostriplen INTEGER; + charlen INTEGER; + + i INTEGER; + tail TEXT; + + trimmed TEXT; +BEGIN + charlen := bit_length('a'); + tostriplen := char_length(tostrip); + expected := tostriplen * charlen; + examined := bit_length(tostrip); + + IF expected = examined OR octets = 0 THEN + RETURN SUBSTRING(tostrip from 1 for (tostriplen - octets)); + ELSIF octets < 0 THEN + RETURN tostrip; + ELSIF (octets * charlen) > examined THEN + RETURN ''; + END IF; + + i := tostriplen - ((octets - 1) / 2); + LOOP + tail := SUBSTRING(tostrip from i for tostriplen); + + EXIT WHEN octet_length(tail) >= octets OR i <= 0; + + i := i - 1; + END LOOP; + + trimmed := SUBSTRING(tostrip from 1 for (tostriplen - char_length(tail))); + RETURN trimmed; +END; +$$ LANGUAGE 'plpgsql'; + From 0d1ba2538b0b035cde654ff3b4c0d3be5ac74a17 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 16:18:46 +0200 Subject: [PATCH 16/36] Adds test for new CDB_Helper 173 --- test/CDB_HelperTest.sql | 64 ++++++++++++++++++++++ test/CDB_HelperTest_expected | 102 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 test/CDB_HelperTest.sql create mode 100644 test/CDB_HelperTest_expected diff --git a/test/CDB_HelperTest.sql b/test/CDB_HelperTest.sql new file mode 100644 index 0000000..10dfb7e --- /dev/null +++ b/test/CDB_HelperTest.sql @@ -0,0 +1,64 @@ +-- Test unique identifier creation with normal length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'relname', NULL); + +-- Test unique identifier creation with prefix with normal length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'relname', NULL); + +-- Test unique identifier creation with suffix with normal length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'relname', '_suffix'); + +-- Test unique identifier creation with long length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); + +-- Test unique identifier creation with prefix with long length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar (name text); +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargolar; + +-- Test unique identifier creation with suffix with long length normal relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix (name text); +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +DROP TABLE largolargolargolargolargolargolargolargolargolargolar_suffix; + +-- Test unique identifier creation with normal length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piraña', NULL); + +-- Test unique identifier creation with prefix with normal length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piraña', NULL); + +-- Test unique identifier creation with suffix with normal length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piraña', '_suffix'); + +-- Test unique identifier creation with long length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); + +-- Test unique identifier creation with prefix with long length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi (name text); +SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +DROP TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi; + +-- Test unique identifier creation with suffix with long length UTF8 relname +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name text); +SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix; + +-- Test CDB_Trim_Octets simple case +SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 1); + +-- Test CDB_Octet_Trim UTF8 case +SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 2); + +-- Test CDB_Octet_Trim UTF8 case +SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 3); diff --git a/test/CDB_HelperTest_expected b/test/CDB_HelperTest_expected new file mode 100644 index 0000000..885a831 --- /dev/null +++ b/test/CDB_HelperTest_expected @@ -0,0 +1,102 @@ + cdb_unique_identifier +----------------------- + relname +(1 row) + + cdb_unique_identifier +----------------------- + prefix_relname +(1 row) + + cdb_unique_identifier +----------------------- + relname_suffix +(1 row) + + cdb_unique_identifier +-------------------------------------------------------------- + largolargolargolargolargolargolargolargolargolargolargolargo +(1 row) + + cdb_unique_identifier +-------------------------------------------------------------- + prefix_largolargolargolargolargolargolargolargolargolargolar +(1 row) + +CREATE TABLE + cdb_unique_identifier +---------------------------------------------------------------- + prefix_largolargolargolargolargolargolargolargolargolargolar_0 +(1 row) + +DROP TABLE + cdb_unique_identifier +-------------------------------------------------------------- + largolargolargolargolargolargolargolargolargolargolar_suffix +(1 row) + +CREATE TABLE + cdb_unique_identifier +---------------------------------------------------------------- + largolargolargolargolargolargolargolargolargolargolar_suffix_0 +(1 row) + +DROP TABLE + cdb_unique_identifier +----------------------- + piraña +(1 row) + + cdb_unique_identifier +----------------------- + prefix_piraña +(1 row) + + cdb_unique_identifier +----------------------- + piraña_suffix +(1 row) + + cdb_unique_identifier +-------------------------------------------------- + piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid +(1 row) + + cdb_unique_identifier +--------------------------------------------------- + prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi +(1 row) + +CREATE TABLE + cdb_unique_identifier +----------------------------------------------------- + prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0 +(1 row) + +DROP TABLE + cdb_unique_identifier +--------------------------------------------------- + piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix +(1 row) + +CREATE TABLE + cdb_unique_identifier +----------------------------------------------------- + piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 +(1 row) + +DROP TABLE + cdb_octet_trim +---------------- + pirañ +(1 row) + + cdb_octet_trim +---------------- + pira +(1 row) + + cdb_octet_trim +---------------- + pira +(1 row) From 8479c3375bab35d9b450f38488fba0a04fce70d7 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 17:01:47 +0200 Subject: [PATCH 17/36] Adds CDB_Unique_Column_Identifier for columns 173 --- scripts-available/CDB_CartodbfyTable.sql | 10 +- scripts-available/CDB_Helper.sql | 65 ++++++++++++- test/CDB_HelperTest_expected | 113 ++++------------------- 3 files changed, 87 insertions(+), 101 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index ace030b..b468928 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -469,7 +469,7 @@ END; $$ LANGUAGE 'plpgsql'; --- DEPRECATED: Use cartodb.CDB_Unique_Identifier since it's UTF8 Safe and length +-- DEPRECATED: Use cartodb.CDB_Unique_Column_Identifier since it's UTF8 Safe and length -- aware. Find a unique column name in the given relation, starting from the -- column name given. If the column name is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. @@ -592,7 +592,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, rec.attname, - cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || const.pkey)), + cartodb.CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -609,7 +609,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', - reloid::text, rec.attname, cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || const.pkey)), + reloid::text, rec.attname, cartodb.CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -775,7 +775,7 @@ BEGIN WHEN others THEN IF SQLERRM = 'parse error - invalid geometry' THEN text_geom_column := false; - str := cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || r1.attname); + str := cartodb.CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', @@ -787,7 +787,7 @@ BEGIN -- Just change its name so we can write a new column into that name. ELSE - str := cartodb.CDB_Unique_Identifier(NULL, reloid::text, '_' || r1.attname); + str := cartodb.CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 013f3ad..687d6cf 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -20,7 +20,7 @@ BEGIN relname := CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); IF relname = '' THEN - PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', 'CDB_Unique_Identifier'); END IF; ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); @@ -52,10 +52,71 @@ BEGIN i := i + 1; END LOOP; - PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier'); + PERFORM _CDB_Error('looping too far', 'CDB_Unique_Identifier'); END; $$ LANGUAGE 'plpgsql'; +-- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix +-- and/or suffix and withing a realtion. If no reloid is give, all relations are examined +CREATE OR REPLACE FUNCTION cartodb.CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL) +RETURNS TEXT +AS $$ +DECLARE + rec RECORD; + usedspace INTEGER; + ident TEXT; + i INTEGER; + origident TEXT; + maxlen INTEGER; +BEGIN + maxlen := 63; + + usedspace := 3; + usedspace := usedspace + coalesce(octet_length(prefix), 0); + usedspace := usedspace + coalesce(octet_length(suffix), 0); + + relname := CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + + IF relname = '' THEN + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', 'CDB_Unique_Identifier'); + END IF; + + ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); + + i := 0; + origident := ident; + + WHILE i < 100 LOOP + IF reloid IS NOT NULL THEN + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE NOT a.attisdropped + AND a.attnum > 0 + AND c.oid = reloid + AND a.attname = ident; + ELSE + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE NOT a.attisdropped + AND a.attnum > 0 + AND a.attname = ident; + END IF; + + IF NOT FOUND THEN + RETURN ident; + END IF; + + ident := origident || '_' || i; + i := i + 1; + END LOOP; + + PERFORM _CDB_Error('looping too far', 'CDB_Unique_Column_Identifier'); +END; +$$ LANGUAGE 'plpgsql'; -- Trims the end of a given string by the given number of octets taking care -- not to leave characters in half. UTF8 safe. diff --git a/test/CDB_HelperTest_expected b/test/CDB_HelperTest_expected index 885a831..acea935 100644 --- a/test/CDB_HelperTest_expected +++ b/test/CDB_HelperTest_expected @@ -1,102 +1,27 @@ - cdb_unique_identifier ------------------------ - relname -(1 row) - - cdb_unique_identifier ------------------------ - prefix_relname -(1 row) - - cdb_unique_identifier ------------------------ - relname_suffix -(1 row) - - cdb_unique_identifier --------------------------------------------------------------- - largolargolargolargolargolargolargolargolargolargolargolargo -(1 row) - - cdb_unique_identifier --------------------------------------------------------------- - prefix_largolargolargolargolargolargolargolargolargolargolar -(1 row) - +relname +prefix_relname +relname_suffix +largolargolargolargolargolargolargolargolargolargolargolargo +prefix_largolargolargolargolargolargolargolargolargolargolar CREATE TABLE - cdb_unique_identifier ----------------------------------------------------------------- - prefix_largolargolargolargolargolargolargolargolargolargolar_0 -(1 row) - +prefix_largolargolargolargolargolargolargolargolargolargolar_0 DROP TABLE - cdb_unique_identifier --------------------------------------------------------------- - largolargolargolargolargolargolargolargolargolargolar_suffix -(1 row) - +largolargolargolargolargolargolargolargolargolargolar_suffix CREATE TABLE - cdb_unique_identifier ----------------------------------------------------------------- - largolargolargolargolargolargolargolargolargolargolar_suffix_0 -(1 row) - +largolargolargolargolargolargolargolargolargolargolar_suffix_0 DROP TABLE - cdb_unique_identifier ------------------------ - piraña -(1 row) - - cdb_unique_identifier ------------------------ - prefix_piraña -(1 row) - - cdb_unique_identifier ------------------------ - piraña_suffix -(1 row) - - cdb_unique_identifier --------------------------------------------------- - piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid -(1 row) - - cdb_unique_identifier ---------------------------------------------------- - prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi -(1 row) - +piraña +prefix_piraña +piraña_suffix +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid +prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi CREATE TABLE - cdb_unique_identifier ------------------------------------------------------ - prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0 -(1 row) - +prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0 DROP TABLE - cdb_unique_identifier ---------------------------------------------------- - piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix -(1 row) - +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix CREATE TABLE - cdb_unique_identifier ------------------------------------------------------ - piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 -(1 row) - +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 DROP TABLE - cdb_octet_trim ----------------- - pirañ -(1 row) - - cdb_octet_trim ----------------- - pira -(1 row) - - cdb_octet_trim ----------------- - pira -(1 row) +pirañ +pira +pira From 974f7f9899f6421398e9a9b8640984489523613e Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 17:08:23 +0200 Subject: [PATCH 18/36] Fixes tests 173 --- test/CDB_CartodbfyTableTest.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index dc4bab7..fc6bd8a 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -291,7 +291,7 @@ INSERT INTO test VALUES (NULL), (3); SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148'); -SELECT cartodb_id, cartodb_id_1 from test; +SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; -- Table with non unique cartodb_id @@ -303,7 +303,7 @@ INSERT INTO test VALUES (2), (2); SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148'); -SELECT cartodb_id, cartodb_id_1 from test; +SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; -- Table with non unique and null cartodb_id @@ -316,7 +316,7 @@ INSERT INTO test VALUES (NULL), (2); SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148'); -SELECT cartodb_id, cartodb_id_1 from test; +SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; From d14436da6c1433ec6c3917b7d10716134e95077c Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 17:14:13 +0200 Subject: [PATCH 19/36] Renames test file 173 --- test/{CDB_HelperTest_expected => CDB_HelperTest_expect} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{CDB_HelperTest_expected => CDB_HelperTest_expect} (100%) diff --git a/test/CDB_HelperTest_expected b/test/CDB_HelperTest_expect similarity index 100% rename from test/CDB_HelperTest_expected rename to test/CDB_HelperTest_expect From 58deeb088dbb06b9bc5a92133f4e870483a9efbe Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 18:56:50 +0200 Subject: [PATCH 20/36] Raises error on deprecated methods. --- scripts-available/CDB_CartodbfyTable.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index b468928..933f386 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -441,6 +441,8 @@ DECLARE newrelname TEXT; BEGIN + RAISE ERROR '_CDB_Unique_Relation_Name is DEPRECATED. Use CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; + i := 0; newrelname := relationname; LOOP @@ -482,6 +484,8 @@ DECLARE newcolname TEXT; BEGIN + RAISE ERROR '_CDB_Unique_Column_Name is DEPRECATED. Use CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; + i := 0; newcolname := columnname; LOOP From aed8671e77f1891715c55a644b318d13c519107a Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 19:00:18 +0200 Subject: [PATCH 21/36] Uses underscore prefix for private tables 173 --- scripts-available/CDB_CartodbfyTable.sql | 20 ++++++++++---------- scripts-available/CDB_Helper.sql | 18 +++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 933f386..2a1e31e 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -428,7 +428,7 @@ END; $$ LANGUAGE 'plpgsql'; --- DEPRECATED: Use cartodb.CDB_Unique_Identifier since it's UTF8 Safe and length +-- DEPRECATED: Use _CDB_Unique_Identifier since it's UTF8 Safe and length -- aware. Find a unique relation name in the given schema, starting from the -- template given. If the template is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. @@ -441,7 +441,7 @@ DECLARE newrelname TEXT; BEGIN - RAISE ERROR '_CDB_Unique_Relation_Name is DEPRECATED. Use CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; + RAISE ERROR '_CDB_Unique_Relation_Name is DEPRECATED. Use _CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; i := 0; newrelname := relationname; @@ -471,7 +471,7 @@ END; $$ LANGUAGE 'plpgsql'; --- DEPRECATED: Use cartodb.CDB_Unique_Column_Identifier since it's UTF8 Safe and length +-- DEPRECATED: Use _CDB_Unique_Column_Identifier since it's UTF8 Safe and length -- aware. Find a unique column name in the given relation, starting from the -- column name given. If the column name is already unique, just return it; -- otherwise, append an increasing integer until you find a unique variant. @@ -484,7 +484,7 @@ DECLARE newcolname TEXT; BEGIN - RAISE ERROR '_CDB_Unique_Column_Name is DEPRECATED. Use CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; + RAISE ERROR '_CDB_Unique_Column_Name is DEPRECATED. Use _CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; i := 0; newcolname := columnname; @@ -596,7 +596,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, rec.attname, - cartodb.CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), + cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -613,7 +613,7 @@ BEGIN PERFORM _CDB_SQL( Format('ALTER TABLE %s RENAME COLUMN %s TO %I', - reloid::text, rec.attname, cartodb.CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), + reloid::text, rec.attname, cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)), '_CDB_Has_Usable_Primary_ID'); END IF; @@ -779,7 +779,7 @@ BEGIN WHEN others THEN IF SQLERRM = 'parse error - invalid geometry' THEN text_geom_column := false; - str := cartodb.CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); + str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', @@ -791,7 +791,7 @@ BEGIN -- Just change its name so we can write a new column into that name. ELSE - str := cartodb.CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); + str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid); sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str); PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom'); RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', @@ -968,14 +968,14 @@ BEGIN -- Put the primary key sequence in the right schema -- If the new table is not moving, better ensure the sequence name -- is unique - destseq := cartodb.CDB_Unique_Identifier(NULL, relname, '_' || const.pkey || '_seq', destschema); + destseq := cartodb._CDB_Unique_Identifier(NULL, relname, '_' || const.pkey || '_seq', destschema); destseq := Format('%I.%I', destschema, destseq); PERFORM _CDB_SQL(Format('CREATE SEQUENCE %s', destseq), '_CDB_Rewrite_Table'); -- Salt a temporary table name if we are re-writing in place -- Note copyname is already escaped and safe to use as identifier IF destschema = relschema THEN - copyname := Format('%I.%I', destschema, cartodb.CDB_Unique_Identifier(NULL, destname, '_' || salt), destschema); + copyname := Format('%I.%I', destschema, cartodb._CDB_Unique_Identifier(NULL, destname, '_' || salt), destschema); ELSE copyname := Format('%I.%I', destschema, destname); END IF; diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 687d6cf..9743a78 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -1,6 +1,6 @@ -- UTF8 safe and lenght aware. Find a unique identifier with a given prefix -- and/or suffix and withing a schema. -CREATE OR REPLACE FUNCTION cartodb.CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL) +CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL) RETURNS TEXT AS $$ DECLARE @@ -17,10 +17,10 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); IF relname = '' THEN - PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', 'CDB_Unique_Identifier'); + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); END IF; ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); @@ -52,13 +52,13 @@ BEGIN i := i + 1; END LOOP; - PERFORM _CDB_Error('looping too far', 'CDB_Unique_Identifier'); + PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier'); END; $$ LANGUAGE 'plpgsql'; -- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix -- and/or suffix and withing a realtion. If no reloid is give, all relations are examined -CREATE OR REPLACE FUNCTION cartodb.CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL) +CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL) RETURNS TEXT AS $$ DECLARE @@ -75,10 +75,10 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); IF relname = '' THEN - PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', 'CDB_Unique_Identifier'); + PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier'); END IF; ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); @@ -114,13 +114,13 @@ BEGIN i := i + 1; END LOOP; - PERFORM _CDB_Error('looping too far', 'CDB_Unique_Column_Identifier'); + PERFORM _CDB_Error('looping too far', '_CDB_Unique_Column_Identifier'); END; $$ LANGUAGE 'plpgsql'; -- Trims the end of a given string by the given number of octets taking care -- not to leave characters in half. UTF8 safe. -CREATE OR REPLACE FUNCTION cartodb.CDB_Octet_Trim(tostrip TEXT, octets INTEGER) +CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Trim(tostrip TEXT, octets INTEGER) RETURNS TEXT AS $$ DECLARE From 9850399693987453d3b9164d5f9965c39a6759d9 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 22 Oct 2015 19:23:38 +0200 Subject: [PATCH 22/36] Adds test for CDB_Unique_Column_Identifier 173 --- test/CDB_HelperTest.sql | 100 +++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/test/CDB_HelperTest.sql b/test/CDB_HelperTest.sql index 10dfb7e..3f82929 100644 --- a/test/CDB_HelperTest.sql +++ b/test/CDB_HelperTest.sql @@ -1,64 +1,120 @@ -- Test unique identifier creation with normal length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'relname', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', NULL); -- Test unique identifier creation with prefix with normal length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'relname', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'relname', NULL); -- Test unique identifier creation with suffix with normal length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'relname', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', '_suffix'); -- Test unique identifier creation with long length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); -- Test unique identifier creation with prefix with long length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); -- Test new identifier is found when name is taken from previous case CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar (name text); -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargolar; -- Test unique identifier creation with suffix with long length normal relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); -- Test new identifier is found when name is taken from previous case CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix (name text); -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); DROP TABLE largolargolargolargolargolargolargolargolargolargolar_suffix; -- Test unique identifier creation with normal length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piraña', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', NULL); -- Test unique identifier creation with prefix with normal length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piraña', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piraña', NULL); -- Test unique identifier creation with suffix with normal length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piraña', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', '_suffix'); -- Test unique identifier creation with long length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); -- Test unique identifier creation with prefix with long length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); -- Test new identifier is found when name is taken from previous case CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi (name text); -SELECT * FROM cartodb.CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); DROP TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi; -- Test unique identifier creation with suffix with long length UTF8 relname -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); -- Test new identifier is found when name is taken from previous case CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name text); -SELECT * FROM cartodb.CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix; --- Test CDB_Trim_Octets simple case -SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 1); +-- Test unique identifier creation with normal length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL); --- Test CDB_Octet_Trim UTF8 case -SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 2); +-- Test unique identifier creation with prefix with normal length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL); --- Test CDB_Octet_Trim UTF8 case -SELECT * FROM cartodb.CDB_Octet_Trim('piraña', 3); +-- Test unique identifier creation with suffix with normal length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix'); + +-- Test unique identifier creation with long length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); + +-- Test unique identifier creation with prefix with long length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargolar text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +DROP TABLE test; + +-- Test unique identifier creation with suffix with long length normal colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE test (largolargolargolargolargolargolargolargolargolargolar_suffix text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +DROP TABLE test; + +-- Test unique identifier creation with normal length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL); + +-- Test unique identifier creation with prefix with normal length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL); + +-- Test unique identifier creation with suffix with normal length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix'); + +-- Test unique identifier creation with long length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); + +-- Test unique identifier creation with prefix with long length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE test (prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +DROP TABLE test; + +-- Test unique identifier creation with suffix with long length UTF8 colname +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); + +-- Test new identifier is found when name is taken from previous case +CREATE TABLE test (piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +DROP TABLE test; + +-- Test _CDB_Trim_Octets simple case +SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 1); + +-- Test _CDB_Octet_Trim UTF8 case +SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 2); + +-- Test _CDB_Octet_Trim UTF8 case +SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 3); \ No newline at end of file From b4acfeca22aabd06fd99251df7f80dce3b07d98c Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 23 Oct 2015 10:38:24 +0200 Subject: [PATCH 23/36] Fixes Typo in throwing exception 173 --- scripts-available/CDB_CartodbfyTable.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 2a1e31e..43aa214 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -441,7 +441,7 @@ DECLARE newrelname TEXT; BEGIN - RAISE ERROR '_CDB_Unique_Relation_Name is DEPRECATED. Use _CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; + RAISE EXCEPTION '_CDB_Unique_Relation_Name is DEPRECATED. Use _CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; i := 0; newrelname := relationname; @@ -484,7 +484,7 @@ DECLARE newcolname TEXT; BEGIN - RAISE ERROR '_CDB_Unique_Column_Name is DEPRECATED. Use _CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; + RAISE EXCEPTION '_CDB_Unique_Column_Name is DEPRECATED. Use _CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; i := 0; newcolname := columnname; From cf8c5e5a33c4ab5f80e99ca68a567e9addce34f7 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 23 Oct 2015 12:11:40 +0200 Subject: [PATCH 24/36] Fixes CDB_HelperTest_expect 173 --- test/CDB_HelperTest_expect | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/CDB_HelperTest_expect b/test/CDB_HelperTest_expect index acea935..d2771d9 100644 --- a/test/CDB_HelperTest_expect +++ b/test/CDB_HelperTest_expect @@ -1,6 +1,9 @@ -relname -prefix_relname -relname_suffix +CREATE TABLE +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 +DROP TABLE +colname +prefix_colname +colname_suffix largolargolargolargolargolargolargolargolargolargolargolargo prefix_largolargolargolargolargolargolargolargolargolargolar CREATE TABLE From 89f5987f535abbee68a671b1ce059f16433f4a53 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 12:02:54 +0100 Subject: [PATCH 25/36] Fix CDB_HelperTest_expect 173 --- test/CDB_HelperTest_expect | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/CDB_HelperTest_expect b/test/CDB_HelperTest_expect index d2771d9..afb7e12 100644 --- a/test/CDB_HelperTest_expect +++ b/test/CDB_HelperTest_expect @@ -1,3 +1,24 @@ +relname +prefix_relname +relname_suffix +largolargolargolargolargolargolargolargolargolargolargolargo +prefix_largolargolargolargolargolargolargolargolargolargolar +CREATE TABLE +prefix_largolargolargolargolargolargolargolargolargolargolar_0 +DROP TABLE +largolargolargolargolargolargolargolargolargolargolar_suffix +CREATE TABLE +largolargolargolargolargolargolargolargolargolargolar_suffix_0 +DROP TABLE +piraña +prefix_piraña +piraña_suffix +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid +prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi +CREATE TABLE +prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0 +DROP TABLE +piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 DROP TABLE From 270d5b314666edc6edbd1839caf2ca2c9a6bc23b Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 15:50:19 +0100 Subject: [PATCH 26/36] Remove code from deprecated methods 173 --- scripts-available/CDB_CartodbfyTable.sql | 50 ------------------------ 1 file changed, 50 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 43aa214..cafc8e9 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -443,30 +443,6 @@ BEGIN RAISE EXCEPTION '_CDB_Unique_Relation_Name is DEPRECATED. Use _CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)'; - i := 0; - newrelname := relationname; - LOOP - - SELECT c.relname, n.nspname - INTO rec - FROM pg_class c - JOIN pg_namespace n ON c.relnamespace = n.oid - WHERE c.relname = newrelname - AND n.nspname = schemaname; - - IF NOT FOUND THEN - RETURN newrelname; - END IF; - - i := i + 1; - newrelname := relationname || '_' || i; - - IF i > 100 THEN - PERFORM _CDB_Error('looping too far', '_CDB_Unique_Relation_Name'); - END IF; - - END LOOP; - END; $$ LANGUAGE 'plpgsql'; @@ -486,32 +462,6 @@ BEGIN RAISE EXCEPTION '_CDB_Unique_Column_Name is DEPRECATED. Use _CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL)'; - i := 0; - newcolname := columnname; - LOOP - - SELECT a.attname - INTO rec - FROM pg_class c - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE NOT a.attisdropped - AND a.attnum > 0 - AND c.oid = reloid - AND a.attname = newcolname; - - IF NOT FOUND THEN - RETURN newcolname; - END IF; - - i := i + 1; - newcolname := columnname || '_' || i; - - IF i > 100 THEN - PERFORM _CDB_Error('looping too far', '_CDB_Unique_Column_Name'); - END IF; - - END LOOP; - END; $$ LANGUAGE 'plpgsql'; From 629214f32f88db7fa279b509f5958ce19554ae2f Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 15:54:08 +0100 Subject: [PATCH 27/36] Adds explanation for SCHEMA DEFAULT NULL 173 --- scripts-available/CDB_Helper.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 9743a78..40d5c02 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -1,5 +1,6 @@ -- UTF8 safe and lenght aware. Find a unique identifier with a given prefix --- and/or suffix and withing a schema. +-- and/or suffix and withing a schema. If a schema is not specified, the identifier +-- is guaranteed to be unique for all schemas. CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL) RETURNS TEXT AS $$ @@ -57,7 +58,7 @@ END; $$ LANGUAGE 'plpgsql'; -- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix --- and/or suffix and withing a realtion. If no reloid is give, all relations are examined +-- and/or suffix and withing a realtion. If no reloid is give, all relations are examined. CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL) RETURNS TEXT AS $$ From bb18d71995335f41c8cf97a36c84b7ee7ce77652 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 16:15:55 +0100 Subject: [PATCH 28/36] Adds comment for magic number and constant for maxlen 173 --- scripts-available/CDB_Helper.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 40d5c02..b24e3c3 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -10,10 +10,10 @@ DECLARE ident TEXT; i INTEGER; origident TEXT; - maxlen INTEGER; -BEGIN - maxlen := 63; + maxlen CONSTANT integer := 63; +BEGIN + -- Accounts for the _XX incremental suffix in case the identifier is taken usedspace := 3; usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); @@ -68,10 +68,10 @@ DECLARE ident TEXT; i INTEGER; origident TEXT; - maxlen INTEGER; -BEGIN - maxlen := 63; + maxlen CONSTANT integer := 63; +BEGIN + -- Accounts for the _XX incremental suffix in case the identifier is taken usedspace := 3; usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); From 710a3c9672ad8e2c6fedd497c8a4a5749cafdd2a Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 16:17:25 +0100 Subject: [PATCH 29/36] Removes ability to examine all relations for _CDB_Unique_Column_Identifier 173 --- scripts-available/CDB_Helper.sql | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index b24e3c3..04d2388 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -58,8 +58,8 @@ END; $$ LANGUAGE 'plpgsql'; -- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix --- and/or suffix and withing a realtion. If no reloid is give, all relations are examined. -CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS DEFAULT NULL) +-- and/or suffix and withing a realtion. +CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS) RETURNS TEXT AS $$ DECLARE @@ -88,24 +88,14 @@ BEGIN origident := ident; WHILE i < 100 LOOP - IF reloid IS NOT NULL THEN - SELECT a.attname - INTO rec - FROM pg_class c - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE NOT a.attisdropped - AND a.attnum > 0 - AND c.oid = reloid - AND a.attname = ident; - ELSE - SELECT a.attname - INTO rec - FROM pg_class c - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE NOT a.attisdropped - AND a.attnum > 0 - AND a.attname = ident; - END IF; + SELECT a.attname + INTO rec + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE NOT a.attisdropped + AND a.attnum > 0 + AND c.oid = reloid + AND a.attname = ident; IF NOT FOUND THEN RETURN ident; From 2e701f73baa0bae19644fa5a58fceed6f6175e0d Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 16:35:33 +0100 Subject: [PATCH 30/36] Reworks _CDB_Octet_Trim 173 --- scripts-available/CDB_Helper.sql | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 04d2388..5227590 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -18,7 +18,7 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Trim_Octets(relname, usedspace + octet_length(relname) - maxlen); IF relname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); @@ -76,7 +76,7 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Octet_Trim(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Trim_Octets(relname, usedspace + octet_length(relname) - maxlen); IF relname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier'); @@ -110,14 +110,15 @@ END; $$ LANGUAGE 'plpgsql'; -- Trims the end of a given string by the given number of octets taking care --- not to leave characters in half. UTF8 safe. -CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Trim(tostrip TEXT, octets INTEGER) +-- not to leave characters in half. If a negative or 0 amount of octects to trim +-- is specified, the suplied text is returned unaltered. UTF8 safe. +CREATE OR REPLACE FUNCTION cartodb._CDB_Trim_Octets(totrim TEXT, octets INTEGER) RETURNS TEXT AS $$ DECLARE expected INTEGER; examined INTEGER; - tostriplen INTEGER; + totrimlen INTEGER; charlen INTEGER; i INTEGER; @@ -126,28 +127,28 @@ DECLARE trimmed TEXT; BEGIN charlen := bit_length('a'); - tostriplen := char_length(tostrip); - expected := tostriplen * charlen; - examined := bit_length(tostrip); + totrimlen := char_length(totrim); + expected := totrimlen * charlen; + examined := bit_length(totrim); - IF expected = examined OR octets = 0 THEN - RETURN SUBSTRING(tostrip from 1 for (tostriplen - octets)); - ELSIF octets < 0 THEN - RETURN tostrip; + IF octets <= 0 THEN + RETURN totrim; + ELSIF expected = examined THEN + RETURN SUBSTRING(totrim from 1 for (totrimlen - octets)); ELSIF (octets * charlen) > examined THEN RETURN ''; END IF; - i := tostriplen - ((octets - 1) / 2); + i := totrimlen - ((octets - 1) / 2); LOOP - tail := SUBSTRING(tostrip from i for tostriplen); + tail := SUBSTRING(totrim from i for totrimlen); EXIT WHEN octet_length(tail) >= octets OR i <= 0; i := i - 1; END LOOP; - trimmed := SUBSTRING(tostrip from 1 for (tostriplen - char_length(tail))); + trimmed := SUBSTRING(totrim from 1 for (totrimlen - char_length(tail))); RETURN trimmed; END; $$ LANGUAGE 'plpgsql'; From bb54eb83c14597db6fed1eb493e1b52094e5effb Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 18:01:03 +0100 Subject: [PATCH 31/36] Drops CDB_Trim_Octets for CDB_Octet_Truncate --- scripts-available/CDB_Helper.sql | 63 ++++++++++++++++---------------- test/CDB_HelperTest.sql | 52 +++++++++++++++----------- test/CDB_HelperTest_expect | 12 +++++- 3 files changed, 71 insertions(+), 56 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 5227590..6338b67 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -5,20 +5,21 @@ CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname T RETURNS TEXT AS $$ DECLARE + maxlen CONSTANT INTEGER := 63; + rec RECORD; usedspace INTEGER; ident TEXT; - i INTEGER; origident TEXT; - maxlen CONSTANT integer := 63; + i INTEGER; BEGIN -- Accounts for the _XX incremental suffix in case the identifier is taken usedspace := 3; usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Trim_Octets(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Octet_Truncate(relname, maxlen - usedspace); IF relname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); @@ -57,26 +58,28 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; + -- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix -- and/or suffix and withing a realtion. CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS) RETURNS TEXT AS $$ DECLARE + maxlen CONSTANT INTEGER := 63; + rec RECORD; usedspace INTEGER; ident TEXT; - i INTEGER; origident TEXT; - maxlen CONSTANT integer := 63; + i INTEGER; BEGIN -- Accounts for the _XX incremental suffix in case the identifier is taken usedspace := 3; usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Trim_Octets(relname, usedspace + octet_length(relname) - maxlen); + relname := _CDB_Octet_Truncate(relname, maxlen - usedspace); IF relname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier'); @@ -109,47 +112,43 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; --- Trims the end of a given string by the given number of octets taking care --- not to leave characters in half. If a negative or 0 amount of octects to trim --- is specified, the suplied text is returned unaltered. UTF8 safe. -CREATE OR REPLACE FUNCTION cartodb._CDB_Trim_Octets(totrim TEXT, octets INTEGER) + +-- Truncates a given string to a max_octets octexts taking care +-- not to leave characters in half. UTF8 safe. +CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Truncate(string TEXT, max_octets INTEGER) RETURNS TEXT AS $$ DECLARE + extcharlen CONSTANT INTEGER := octet_length('ñ'); + expected INTEGER; examined INTEGER; - totrimlen INTEGER; - charlen INTEGER; + strlen INTEGER; i INTEGER; - tail TEXT; - - trimmed TEXT; BEGIN - charlen := bit_length('a'); - totrimlen := char_length(totrim); - expected := totrimlen * charlen; - examined := bit_length(totrim); - IF octets <= 0 THEN - RETURN totrim; - ELSIF expected = examined THEN - RETURN SUBSTRING(totrim from 1 for (totrimlen - octets)); - ELSIF (octets * charlen) > examined THEN + IF max_octets <= 0 THEN RETURN ''; + ELSIF max_octets >= octet_length(string) THEN + RETURN string; END IF; - i := totrimlen - ((octets - 1) / 2); - LOOP - tail := SUBSTRING(totrim from i for totrimlen); + strlen := char_length(string); - EXIT WHEN octet_length(tail) >= octets OR i <= 0; + expected := char_length(string); + examined := octet_length(string); - i := i - 1; + IF expected = examined THEN + RETURN SUBSTRING(string from 1 for max_octets); + END IF; + + i := max_octets / extcharlen; + + WHILE octet_length(SUBSTRING(string from 1 for i)) <= max_octets LOOP + i := i + 1; END LOOP; - trimmed := SUBSTRING(totrim from 1 for (totrimlen - char_length(tail))); - RETURN trimmed; + RETURN SUBSTRING(string from 1 for (i - 1)); END; $$ LANGUAGE 'plpgsql'; - diff --git a/test/CDB_HelperTest.sql b/test/CDB_HelperTest.sql index 3f82929..86f1e90 100644 --- a/test/CDB_HelperTest.sql +++ b/test/CDB_HelperTest.sql @@ -54,67 +54,75 @@ CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name t SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix; +CREATE TABLE test (name text); -- Test unique identifier creation with normal length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL, 'test'::regclass); -- Test unique identifier creation with prefix with normal length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL, 'test'::regclass); -- Test unique identifier creation with suffix with normal length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix', 'test'::regclass); -- Test unique identifier creation with long length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass); -- Test unique identifier creation with prefix with long length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass); +DROP TABLE test; -- Test new identifier is found when name is taken from previous case CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargolar text); -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass); DROP TABLE test; -- Test unique identifier creation with suffix with long length normal colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +CREATE TABLE test (name text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass); +DROP TABLE test; -- Test new identifier is found when name is taken from previous case CREATE TABLE test (largolargolargolargolargolargolargolargolargolargolar_suffix text); -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass); DROP TABLE test; +CREATE TABLE test (name text); -- Test unique identifier creation with normal length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL, 'test'::regclass); -- Test unique identifier creation with prefix with normal length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL, 'test'::regclass); -- Test unique identifier creation with suffix with normal length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix', 'test'::regclass); -- Test unique identifier creation with long length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass); -- Test unique identifier creation with prefix with long length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass); +DROP TABLE test; -- Test new identifier is found when name is taken from previous case CREATE TABLE test (prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi text); -SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass); DROP TABLE test; -- Test unique identifier creation with suffix with long length UTF8 colname -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +CREATE TABLE test (name text); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass); +DROP TABLE test; -- Test new identifier is found when name is taken from previous case CREATE TABLE test (piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix text); -SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix'); +SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass); DROP TABLE test; --- Test _CDB_Trim_Octets simple case -SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 1); +-- Test _CDB_Octet_Truncate simple case +SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 5); --- Test _CDB_Octet_Trim UTF8 case -SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 2); +-- Test _CDB_Octet_Truncate UTF8 case +SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 6); --- Test _CDB_Octet_Trim UTF8 case -SELECT * FROM cartodb._CDB_Octet_Trim('piraña', 3); \ No newline at end of file +-- Test _CDB_Octet_Truncate UTF8 case +SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 7); diff --git a/test/CDB_HelperTest_expect b/test/CDB_HelperTest_expect index afb7e12..1c1acf2 100644 --- a/test/CDB_HelperTest_expect +++ b/test/CDB_HelperTest_expect @@ -22,30 +22,38 @@ piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 DROP TABLE +CREATE TABLE colname prefix_colname colname_suffix largolargolargolargolargolargolargolargolargolargolargolargo prefix_largolargolargolargolargolargolargolargolargolargolar +DROP TABLE CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar_0 DROP TABLE +CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix +DROP TABLE CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix_0 DROP TABLE +CREATE TABLE piraña prefix_piraña piraña_suffix piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi +DROP TABLE CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0 DROP TABLE +CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix +DROP TABLE CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0 DROP TABLE +pira pirañ -pira -pira +piraña From 8734608792073316a856e1b8a53b984d2d296be5 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 18:25:48 +0100 Subject: [PATCH 32/36] Removes unnecessary salt 173 --- scripts-available/CDB_CartodbfyTable.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index cafc8e9..849238e 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -811,8 +811,7 @@ DECLARE destname TEXT; destseq TEXT; destseqmax INTEGER; - - salt TEXT := md5(random()::text || now()); + copyname TEXT; column_name_sql TEXT; @@ -922,10 +921,10 @@ BEGIN destseq := Format('%I.%I', destschema, destseq); PERFORM _CDB_SQL(Format('CREATE SEQUENCE %s', destseq), '_CDB_Rewrite_Table'); - -- Salt a temporary table name if we are re-writing in place + -- Temporary table name if we are re-writing in place -- Note copyname is already escaped and safe to use as identifier IF destschema = relschema THEN - copyname := Format('%I.%I', destschema, cartodb._CDB_Unique_Identifier(NULL, destname, '_' || salt), destschema); + copyname := Format('%I.%I', destschema, cartodb._CDB_Unique_Identifier(NULL, destname, NULL), destschema); ELSE copyname := Format('%I.%I', destschema, destname); END IF; From 58b2705383197951d1c41ee82a904cbda80780be Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 26 Oct 2015 18:35:12 +0100 Subject: [PATCH 33/36] Rename relname to colname in _CDB_Unique_Column_Identifier --- scripts-available/CDB_Helper.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 6338b67..3cf97ff 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -60,8 +60,8 @@ $$ LANGUAGE 'plpgsql'; -- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix --- and/or suffix and withing a realtion. -CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, relname TEXT, suffix TEXT, reloid REGCLASS) +-- and/or suffix based on colname and within a relation specified via reloid. +CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, colname TEXT, suffix TEXT, reloid REGCLASS) RETURNS TEXT AS $$ DECLARE @@ -79,13 +79,13 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Octet_Truncate(relname, maxlen - usedspace); + colname := _CDB_Octet_Truncate(colname, maxlen - usedspace); - IF relname = '' THEN + IF colname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier'); END IF; - ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); + ident := coalesce(prefix, '') || colname || coalesce(suffix, ''); i := 0; origident := ident; From 32c729e464e694f71a81cea353041c23e0ac963f Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 27 Oct 2015 10:33:15 +0100 Subject: [PATCH 34/36] Refactors code 173 --- scripts-available/CDB_Helper.sql | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 3cf97ff..483ba0c 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -11,6 +11,7 @@ DECLARE usedspace INTEGER; ident TEXT; origident TEXT; + candrelname TEXT; i INTEGER; BEGIN @@ -19,13 +20,13 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - relname := _CDB_Octet_Truncate(relname, maxlen - usedspace); + candrelname := _CDB_Octet_Truncate(relname, maxlen - usedspace); - IF relname = '' THEN + IF candrelname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier'); END IF; - ident := coalesce(prefix, '') || relname || coalesce(suffix, ''); + ident := coalesce(prefix, '') || candrelname || coalesce(suffix, ''); i := 0; origident := ident; @@ -68,6 +69,7 @@ DECLARE maxlen CONSTANT INTEGER := 63; rec RECORD; + candcolname TEXT; usedspace INTEGER; ident TEXT; origident TEXT; @@ -79,13 +81,13 @@ BEGIN usedspace := usedspace + coalesce(octet_length(prefix), 0); usedspace := usedspace + coalesce(octet_length(suffix), 0); - colname := _CDB_Octet_Truncate(colname, maxlen - usedspace); + candcolname := _CDB_Octet_Truncate(colname, maxlen - usedspace); - IF colname = '' THEN + IF candcolname = '' THEN PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier'); END IF; - ident := coalesce(prefix, '') || colname || coalesce(suffix, ''); + ident := coalesce(prefix, '') || candcolname || coalesce(suffix, ''); i := 0; origident := ident; @@ -140,15 +142,15 @@ BEGIN examined := octet_length(string); IF expected = examined THEN - RETURN SUBSTRING(string from 1 for max_octets); + RETURN left(string, max_octets); END IF; i := max_octets / extcharlen; - WHILE octet_length(SUBSTRING(string from 1 for i)) <= max_octets LOOP + WHILE octet_length(left(string, i)) <= max_octets LOOP i := i + 1; END LOOP; - RETURN SUBSTRING(string from 1 for (i - 1)); + RETURN left(string, (i - 1)); END; $$ LANGUAGE 'plpgsql'; From fdfbe8e62cde3425627b1b1e954b69269e1764e6 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 27 Oct 2015 12:11:00 +0100 Subject: [PATCH 35/36] Typo fixing 173 --- scripts-available/CDB_Helper.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_Helper.sql b/scripts-available/CDB_Helper.sql index 483ba0c..5540086 100644 --- a/scripts-available/CDB_Helper.sql +++ b/scripts-available/CDB_Helper.sql @@ -1,4 +1,4 @@ --- UTF8 safe and lenght aware. Find a unique identifier with a given prefix +-- UTF8 safe and length aware. Find a unique identifier with a given prefix -- and/or suffix and withing a schema. If a schema is not specified, the identifier -- is guaranteed to be unique for all schemas. CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL) @@ -60,7 +60,7 @@ END; $$ LANGUAGE 'plpgsql'; --- UTF8 safe and lenght aware. Find a unique identifier for a column with a given prefix +-- UTF8 safe and length aware. Find a unique identifier for a column with a given prefix -- and/or suffix based on colname and within a relation specified via reloid. CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, colname TEXT, suffix TEXT, reloid REGCLASS) RETURNS TEXT @@ -115,7 +115,7 @@ END; $$ LANGUAGE 'plpgsql'; --- Truncates a given string to a max_octets octexts taking care +-- Truncates a given string to a max_octets octets taking care -- not to leave characters in half. UTF8 safe. CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Truncate(string TEXT, max_octets INTEGER) RETURNS TEXT From d171afb9f4f19d9cd93a13c21b5be827c7acc58a Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Tue, 27 Oct 2015 14:16:17 +0100 Subject: [PATCH 36/36] Increase version number and update NEWS.md --- Makefile | 3 ++- NEWS.md | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b912941..c408057 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.11.2 +EXTVERSION = 0.11.3 SED = sed @@ -54,6 +54,7 @@ UPGRADABLE = \ 0.11.0 \ 0.11.1 \ 0.11.2 \ + 0.11.3 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index 42713d2..dc3fcc9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,13 @@ next (2015-mm-dd) ----------------- +0.11.3 (2015-10-27) +------------------- +* Added CDB_Helper.sql [#173](https://github.com/CartoDB/cartodb-postgresql/pull/173) +* Added _CDB_Unique_Identifier for creating UTF8 aware unique identifiers +* Added _CDB_Unique_Column_Identifier for creating UTF8 aware unique identifiers for columns +* Added _CDB_Octet_Truncate that truncates text to a certain amount of octets. + 0.11.2 (2015-10-19) ------------------- * Fix schema not being specified on pg_get_serial_sequence [#170](https://github.com/CartoDB/cartodb-postgresql/pull/170)