From fc0e883c202fedc6a29e6ea31ced446f45ae822c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 15 Sep 2015 11:51:27 +0200 Subject: [PATCH 01/20] First version of the cartodbfy requirements --- doc/cartodbfy-requirements.rst | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 doc/cartodbfy-requirements.rst diff --git a/doc/cartodbfy-requirements.rst b/doc/cartodbfy-requirements.rst new file mode 100644 index 0000000..4492a23 --- /dev/null +++ b/doc/cartodbfy-requirements.rst @@ -0,0 +1,59 @@ +Introduction +============ + +This document aims at describing what cartodbfy is and what its formal requirements are, with the following goals in mind: + +- clarify what are the expectations of the "cartodbfycation process". +- define an important part of what should be a stable, public API +- allow for better testing, which should in turn... +- ...ease modifications and increase quality of the code + + + +What is the cartodbfycation +=========================== + +The cartodbfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis. + + + +Valid CartoDB tables +==================== + +A valid CartoDB table shall meet the following conditions: + +- Have a ``cartodb_id`` column as primary key with a sequence +- Have a ``the_geom`` column of type ``Geometry`` with SRID 4326 +- Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857 +- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync + +Additionally, a CartoDB table can contain other columns. + + + +High level requirements +======================= + +Here is a list of high level requirments for the public function ``CDB_CartodbfyTable()``: + +- A call to ``CDB_CartodbfyTable()`` shall modify/rewrite the table and produce a valid CartoDB table with the same name. +- A call to ``CDB_CartodbfyTable()`` shall cause the registration of the table into the platform +- It shall be idempotent, meaning that successive calls to ``CDB_CartodbfyTable()`` shall not produce any visible effect in the system. +- If there's a column containing a geometry, it shall be used to generate ``the_geom`` and the ``the_geom_webmercator`` columns. +- Exporting and re-importing the same table in CartoDB shall produce equivalent tables, with the same features associated to the same ``cartodb_id``'s. + +Note that there should be only one feature per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined. + + + +Low-level requirements +====================== + +- If the original table contains a valid ``cartodb_id`` column, it shall be used +- If the original table contains a valid ``the_geom`` or ``the_geom_webmercator`` it shall be used. +- A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata`` +- A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id`` +- A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator`` +- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns + + From eb475fe55f87360c56230273ac16b9152d82aa30 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 14 Oct 2015 11:36:48 -0400 Subject: [PATCH 02/20] Adding function to produce a great circle between two points. --- doc/CDB_GreatCircle.md | 16 ++++++++++++++ scripts-available/CDB_GreatCircle.sql | 30 +++++++++++++++++++++++++++ scripts-enabled/DB_GreatCircle.sql | 1 + test/CDB_GreatCircle.expect | 1 + test/CDB_GreatCircle.sql | 1 + 5 files changed, 49 insertions(+) create mode 100644 doc/CDB_GreatCircle.md create mode 100644 scripts-available/CDB_GreatCircle.sql create mode 120000 scripts-enabled/DB_GreatCircle.sql create mode 100644 test/CDB_GreatCircle.expect create mode 100644 test/CDB_GreatCircle.sql diff --git a/doc/CDB_GreatCircle.md b/doc/CDB_GreatCircle.md new file mode 100644 index 0000000..e32c5b3 --- /dev/null +++ b/doc/CDB_GreatCircle.md @@ -0,0 +1,16 @@ +Based on Paul Ramsey's [blog post](http://blog.cartodb.com/jets-and-datelines/). +#### Using the function + +Creates a great circle line. + +```sql +SELECT CDB_GreatCircle(start_point, end_point) FROM table_name +-- Results a line reprsenting the great circle between the two points +``` + +#### Arguments + +CDB_GreatCircle(start_point, end_point) + +* **start_point** ST_Point indicating the start of the line. +* **end_point** ST_point indicating the end of the line. diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql new file mode 100644 index 0000000..aac2e77 --- /dev/null +++ b/scripts-available/CDB_GreatCircle.sql @@ -0,0 +1,30 @@ +-- Return a line for the Great Circle between two points +-- +-- start_point geometry : The origin of the line +-- end_point geometry : The distination of the line +-- retruns geometry +create or replace function CDB_GreatCircle(start_point geometry ,end_point geometry ) RETURNS geometry as +$$ +DECLARE + line geometry; +BEGIN + +line = ST_Segmentize( + ST_Makeline( + start_point, + end_point + )::geography, + 100000 + )::geometry; + +if ST_XMax(line) - ST_XMin(line) > 180 then + + line = ST_Difference( + ST_Shift_Longitude(line), ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)',4326), 0.00001)); +end if; + + +return line; + +END $$ +LANGUAGE 'plpgsql' diff --git a/scripts-enabled/DB_GreatCircle.sql b/scripts-enabled/DB_GreatCircle.sql new file mode 120000 index 0000000..691bb2d --- /dev/null +++ b/scripts-enabled/DB_GreatCircle.sql @@ -0,0 +1 @@ +scripts-available/CDB_GreatCircle.sql \ No newline at end of file diff --git a/test/CDB_GreatCircle.expect b/test/CDB_GreatCircle.expect new file mode 100644 index 0000000..0b9f5c7 --- /dev/null +++ b/test/CDB_GreatCircle.expect @@ -0,0 +1 @@ +LINESTRING(4.259 55.858,5.6692453115051 56.0150275120673,7.10720375678704 56.157400475677,8.5718366560563 56.2842986378254,10.0619272412891 56.3949153508462,11.5760785994189 56.4884642014437,13.1127142001617 56.564185934303,14.6700812655504 56.6213555706215,16.2462571744128 56.6592896061102,17.8391590143095 56.6773531596105,19.4465562981665 56.6749669334121,21.0660867567155 56.6516138405427,22.6952750058883 56.6068451534252,24.3315537765309 56.540286032869,25.9722872888145 56.4516403065472,27.6147962622065 56.3406943817481,29.2563839799455 56.207320197769,30.8943627796619 56.0514771479657,32.5260803224591 55.8732129290618,34.1489450028345 55.6726633044968,35.7604499005266 55.4500507979281,37.3581947399686 55.2056823610616,38.9399054089486 54.9399460854786,40.5034506895044 54.6533070499613,42.0468559644411 54.3463024122038,43.5683137754523 54.0195358662507,45.066191217402 53.673671594382,46.5390342525062 53.3094278446298,47.9855691138079 52.9275702630659,49.4047010366934 52.5289051040742,50.7955106088955 52.1142724327331,52.1572480633875 51.6845394219955,53.4893258557794 51.2405938343407,54.7913098701049 50.7833377637432,56.0629095865715 50.3136816997865,57.3039675245588 49.8325389621027,58.5144482465496 49.3408205404538,59.6944271762829 48.8394303639846,60.8440794494795 48.329261012675,61.9636689799149 47.811189874886,63.0535378889196 47.2860757471582,64.1140964137264 46.7547558660385,65.1458133802427 46.2180433565989,66.1492072992903 45.676725078361,67.1248381223566 45.131559846414,68.0732996734468 44.583277003498,68.9952127576034 44.0325753175597,69.8912189338183 43.4801221786776,70.7619749300985 42.9265530691612,71.6081476710291 42.3724712809595,72.4304098829428 41.8184478551838,73.2294362384225 41.2650217194684,74.0059 40.7127) diff --git a/test/CDB_GreatCircle.sql b/test/CDB_GreatCircle.sql new file mode 100644 index 0000000..496d2cb --- /dev/null +++ b/test/CDB_GreatCircle.sql @@ -0,0 +1 @@ +select CDB_GreatCircle(CDB_LATLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)) From 2908c270b377acc38728fc1f61f9fc7dabd9fbf3 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 14 Oct 2015 11:48:08 -0400 Subject: [PATCH 03/20] Fixing typo with symlink --- scripts-enabled/241-CDB_GreatCircle.sql | 1 + scripts-enabled/DB_GreatCircle.sql | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 scripts-enabled/241-CDB_GreatCircle.sql delete mode 120000 scripts-enabled/DB_GreatCircle.sql diff --git a/scripts-enabled/241-CDB_GreatCircle.sql b/scripts-enabled/241-CDB_GreatCircle.sql new file mode 120000 index 0000000..8bec24c --- /dev/null +++ b/scripts-enabled/241-CDB_GreatCircle.sql @@ -0,0 +1 @@ +../scripts-available/CDB_GreatCircle.sql \ No newline at end of file diff --git a/scripts-enabled/DB_GreatCircle.sql b/scripts-enabled/DB_GreatCircle.sql deleted file mode 120000 index 691bb2d..0000000 --- a/scripts-enabled/DB_GreatCircle.sql +++ /dev/null @@ -1 +0,0 @@ -scripts-available/CDB_GreatCircle.sql \ No newline at end of file From 66249843e8d279d02caf31f1473f490ab513f29a Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 14 Oct 2015 12:12:12 -0400 Subject: [PATCH 04/20] making some text lowercase --- test/CDB_GreatCircle.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CDB_GreatCircle.sql b/test/CDB_GreatCircle.sql index 496d2cb..c9bfcb6 100644 --- a/test/CDB_GreatCircle.sql +++ b/test/CDB_GreatCircle.sql @@ -1 +1 @@ -select CDB_GreatCircle(CDB_LATLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)) +select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059))) From 453b3af8720ec1f94afef586e923daa7c4d8f984 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 30 Oct 2015 17:49:27 +0100 Subject: [PATCH 05/20] Removes extra d in delimiter 155 --- scripts-available/CDB_CartodbfyTable.sql | 2 +- test/CDB_CartodbfyTableTest.sql | 7 +++++++ test/CDB_CartodbfyTableTest_expect | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 849238e..172c372 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -166,7 +166,7 @@ BEGIN AS seq INTO rec2; -- Reset sequence name - sql := Format('ALTER SEQUENCE %s RESTART WITH %d', rec2.seq::text, rec.max + 1); + sql := Format('ALTER SEQUENCE %s RESTART WITH %', rec2.seq::text, rec.max + 1); RAISE DEBUG 'Running %', sql; EXECUTE sql; diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index fc6bd8a..280e8c5 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -319,6 +319,13 @@ SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_i SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; +-- _CDB_create_cartodb_id_column with cartodb_id already present +CREATE TABLE test (cartodb_id integer); + +SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass); +SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0'; + +DROP TABLE test; -- TODO: table with existing custom-triggered the_geom diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 6103dc8..0e866e9 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -102,5 +102,9 @@ Table with non unique and null cartodb_id #148 cartodbfied fine 3| 4|2 DROP TABLE +CREATE TABLE + +_cartodb_id0 +DROP TABLE DROP FUNCTION DROP FUNCTION From 081ed36aae65e3fb78499378b28a77a9b279570a Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Mon, 2 Nov 2015 11:47:14 +0100 Subject: [PATCH 06/20] Fixes NULL max in rec 155 --- scripts-available/CDB_CartodbfyTable.sql | 4 ++-- test/CDB_CartodbfyTableTest.sql | 10 +++++++++- test/CDB_CartodbfyTableTest_expect | 3 +++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 172c372..44127c7 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -157,7 +157,7 @@ BEGIN EXECUTE sql; -- Find max value - sql := Format('SELECT max(cartodb_id) FROM %s', reloid::text); + sql := Format('SELECT coalesce(max(cartodb_id), 0) as max FROM %s', reloid::text); RAISE DEBUG 'Running %', sql; EXECUTE sql INTO rec; @@ -166,7 +166,7 @@ BEGIN AS seq INTO rec2; -- Reset sequence name - sql := Format('ALTER SEQUENCE %s RESTART WITH %', rec2.seq::text, rec.max + 1); + sql := Format('ALTER SEQUENCE %s RESTART WITH %s', rec2.seq::text, rec.max + 1); RAISE DEBUG 'Running %', sql; EXECUTE sql; diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 280e8c5..367f31a 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -319,7 +319,7 @@ SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_i SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; --- _CDB_create_cartodb_id_column with cartodb_id already present +-- _CDB_create_cartodb_id_column with cartodb_id integer already present CREATE TABLE test (cartodb_id integer); SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass); @@ -327,6 +327,14 @@ SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND DROP TABLE test; +-- _CDB_create_cartodb_id_column with cartodb_id text already present +CREATE TABLE test (cartodb_id text); + +SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass); +SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0'; + +DROP TABLE test; + -- TODO: table with existing custom-triggered the_geom DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text); diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 0e866e9..6317b33 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -104,6 +104,9 @@ Table with non unique and null cartodb_id #148 cartodbfied fine DROP TABLE CREATE TABLE +DROP TABLE +CREATE TABLE + _cartodb_id0 DROP TABLE DROP FUNCTION From 6252907de28e84d310b79e8006b8b29f1eb6822d Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Thu, 5 Nov 2015 17:11:27 +0100 Subject: [PATCH 07/20] Rises verbosity level before tests for _CDB_create_cartodb_id_column 155 --- test/CDB_CartodbfyTableTest.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 367f31a..5614786 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -319,6 +319,7 @@ SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_i SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; +\set VERBOSITY terse -- _CDB_create_cartodb_id_column with cartodb_id integer already present CREATE TABLE test (cartodb_id integer); @@ -334,6 +335,7 @@ SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass); SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0'; DROP TABLE test; +\set VERBOSITY default -- TODO: table with existing custom-triggered the_geom From 545196811f598c6262fb52e89f369d641f9dd60a Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 6 Nov 2015 14:41:12 +0100 Subject: [PATCH 08/20] Changes logger level to error 155 --- test/CDB_CartodbfyTableTest.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 5614786..76e62b0 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -319,7 +319,7 @@ SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_i SELECT cartodb_id, cartodb_id_0 from test; DROP TABLE test; -\set VERBOSITY terse +SET client_min_messages TO notice; -- _CDB_create_cartodb_id_column with cartodb_id integer already present CREATE TABLE test (cartodb_id integer); @@ -335,7 +335,7 @@ SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass); SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0'; DROP TABLE test; -\set VERBOSITY default +SET client_min_messages TO error; -- TODO: table with existing custom-triggered the_geom From 7cf0d02935831302683ae34ca992310d1c54e5cd Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 6 Nov 2015 15:11:08 +0100 Subject: [PATCH 09/20] Fixes expected 155 --- test/CDB_CartodbfyTableTest_expect | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 6317b33..5c464ad 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -102,12 +102,21 @@ Table with non unique and null cartodb_id #148 cartodbfied fine 3| 4|2 DROP TABLE +SET CREATE TABLE +NOTICE: Column cartodb_id already exists +NOTICE: Existing cartodb_id field does not have an associated sequence, renaming +NOTICE: Trying to recover data from _cartodb_id0 column DROP TABLE CREATE TABLE +NOTICE: Column cartodb_id already exists +NOTICE: Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming +NOTICE: Trying to recover data from _cartodb_id0 column +NOTICE: Could not initialize cartodb_id with existing values: result of USING clause for column "cartodb_id" cannot be cast automatically to type integer (42804) _cartodb_id0 DROP TABLE +SET DROP FUNCTION DROP FUNCTION From cce63f0eaeec8be4d0b416e1d492adf5ae124ab2 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 6 Nov 2015 16:26:03 +0100 Subject: [PATCH 10/20] Adds ::integer cast for USING 155 --- scripts-available/CDB_CartodbfyTable.sql | 2 +- test/CDB_CartodbfyTableTest_expect | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts-available/CDB_CartodbfyTable.sql b/scripts-available/CDB_CartodbfyTable.sql index 44127c7..4d44c22 100644 --- a/scripts-available/CDB_CartodbfyTable.sql +++ b/scripts-available/CDB_CartodbfyTable.sql @@ -152,7 +152,7 @@ BEGIN -- Copy existing values to new field -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost - sql := Format('ALTER TABLE %s ALTER cartodb_id TYPE int USING %I', reloid::text, new_name); + sql := Format('ALTER TABLE %s ALTER cartodb_id TYPE int USING %I::integer', reloid::text, new_name); RAISE DEBUG 'Running %', sql; EXECUTE sql; diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 5c464ad..a3057fb 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -113,7 +113,6 @@ CREATE TABLE NOTICE: Column cartodb_id already exists NOTICE: Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming NOTICE: Trying to recover data from _cartodb_id0 column -NOTICE: Could not initialize cartodb_id with existing values: result of USING clause for column "cartodb_id" cannot be cast automatically to type integer (42804) _cartodb_id0 DROP TABLE From 4803abf365f83d12d5a4ab3852e149994df1ba69 Mon Sep 17 00:00:00 2001 From: Guido Fioravantti Date: Fri, 6 Nov 2015 16:32:56 +0100 Subject: [PATCH 11/20] Fix typo in expected 155 --- test/CDB_CartodbfyTableTest_expect | 1 - 1 file changed, 1 deletion(-) diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index a3057fb..e63dbd3 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -114,7 +114,6 @@ NOTICE: Column cartodb_id already exists NOTICE: Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming NOTICE: Trying to recover data from _cartodb_id0 column -_cartodb_id0 DROP TABLE SET DROP FUNCTION From f36f1ab5368d53a7565259658f69657fba1b2832 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 18 Nov 2015 15:30:16 +0000 Subject: [PATCH 12/20] formatting --- scripts-available/CDB_GreatCircle.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index aac2e77..53b163c 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -1,8 +1,11 @@ +-- -- Return a line for the Great Circle between two points -- -- start_point geometry : The origin of the line -- end_point geometry : The distination of the line -- retruns geometry +-- + create or replace function CDB_GreatCircle(start_point geometry ,end_point geometry ) RETURNS geometry as $$ DECLARE From d597f0fe6d9cb2ae2eb5401ca5e63e71a1fd20a4 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 18 Nov 2015 15:56:31 +0000 Subject: [PATCH 13/20] removing comments --- scripts-available/CDB_GreatCircle.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index 53b163c..5d44b51 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -1,11 +1,3 @@ --- --- Return a line for the Great Circle between two points --- --- start_point geometry : The origin of the line --- end_point geometry : The distination of the line --- retruns geometry --- - create or replace function CDB_GreatCircle(start_point geometry ,end_point geometry ) RETURNS geometry as $$ DECLARE From 333a40819958f71ab4650a3c279a2cd1f35be850 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 20 Nov 2015 17:15:37 +0100 Subject: [PATCH 14/20] Add Paul and Rafa feedback --- doc/cartodbfy-requirements.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/cartodbfy-requirements.rst b/doc/cartodbfy-requirements.rst index 4492a23..e3fa3a5 100644 --- a/doc/cartodbfy-requirements.rst +++ b/doc/cartodbfy-requirements.rst @@ -22,7 +22,7 @@ Valid CartoDB tables A valid CartoDB table shall meet the following conditions: -- Have a ``cartodb_id`` column as primary key with a sequence +- Have a ``cartodb_id`` integer column as primary key with a sequence as default value - Have a ``the_geom`` column of type ``Geometry`` with SRID 4326 - Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857 - The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync @@ -49,8 +49,8 @@ Note that there should be only one feature per row in the source table. If there Low-level requirements ====================== -- If the original table contains a valid ``cartodb_id`` column, it shall be used -- If the original table contains a valid ``the_geom`` or ``the_geom_webmercator`` it shall be used. +- If the original table contains a valid (unique) ``cartodb_id`` column, it shall be used +- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used. - A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata`` - A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id`` - A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator`` From bcfe8d8f3b7e9470190a5871f555fe7ca90ad088 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 20 Nov 2015 17:24:01 +0100 Subject: [PATCH 15/20] Adds not null constraint for cartodb_id --- doc/cartodbfy-requirements.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/cartodbfy-requirements.rst b/doc/cartodbfy-requirements.rst index e3fa3a5..0743699 100644 --- a/doc/cartodbfy-requirements.rst +++ b/doc/cartodbfy-requirements.rst @@ -49,7 +49,7 @@ Note that there should be only one feature per row in the source table. If there Low-level requirements ====================== -- If the original table contains a valid (unique) ``cartodb_id`` column, it shall be used +- If the original table contains a valid (unique and not null) ``cartodb_id`` column, it shall be used - If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used. - A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata`` - A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id`` From 3bc92d40464780ed5f6ba68f4c6d7c32c0c48129 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 Nov 2015 16:40:42 +0100 Subject: [PATCH 16/20] Fix CDB_GreatCircle syntax --- scripts-available/CDB_GreatCircle.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index 5d44b51..ea1fa00 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -21,5 +21,5 @@ end if; return line; -END $$ -LANGUAGE 'plpgsql' +END; $$ +LANGUAGE 'plpgsql'; From 2d13903d5089bf002e233ef5bae2b4ba18a30415 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 Nov 2015 16:51:34 +0100 Subject: [PATCH 17/20] Cleanup coding style --- scripts-available/CDB_GreatCircle.sql | 40 +++++++++++++-------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index ea1fa00..004ae2e 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -1,25 +1,23 @@ -create or replace function CDB_GreatCircle(start_point geometry ,end_point geometry ) RETURNS geometry as -$$ +CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry) +RETURNS geometry AS $$ DECLARE - line geometry; + line geometry; BEGIN + line = ST_Segmentize( + ST_Makeline( + start_point, + end_point + )::geography, + 100000 + )::geometry; -line = ST_Segmentize( - ST_Makeline( - start_point, - end_point - )::geography, - 100000 - )::geometry; - -if ST_XMax(line) - ST_XMin(line) > 180 then - - line = ST_Difference( - ST_Shift_Longitude(line), ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)',4326), 0.00001)); -end if; - - -return line; - -END; $$ + IF ST_XMax(line) - ST_XMin(line) > 180 THEN + line = ST_Difference( + ST_Shift_Longitude(line), + ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001) + ); + END IF; +RETURN line; +END; +$$ LANGUAGE 'plpgsql'; From 1b8ced22a5a11a2016c15d2332f8d87959d7a3d2 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 Nov 2015 16:57:34 +0100 Subject: [PATCH 18/20] Fix name of expected results file --- test/{CDB_GreatCircle.expect => CDB_GreatCircle_expect} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{CDB_GreatCircle.expect => CDB_GreatCircle_expect} (100%) diff --git a/test/CDB_GreatCircle.expect b/test/CDB_GreatCircle_expect similarity index 100% rename from test/CDB_GreatCircle.expect rename to test/CDB_GreatCircle_expect From aa81c6a1aba418e2ce7164b38b1fb574bcb6dab2 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 Nov 2015 17:07:24 +0100 Subject: [PATCH 19/20] Add comment to CDB_GreatCircle --- scripts-available/CDB_GreatCircle.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index 004ae2e..8d98dbf 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -1,3 +1,6 @@ +-- Great circle point-to-point routes, based on: +-- http://blog.cartodb.com/jets-and-datelines/ +-- CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry) RETURNS geometry AS $$ DECLARE From f14fc057e2a7b991443020741b7aa93b0fc8c758 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 Nov 2015 17:13:17 +0100 Subject: [PATCH 20/20] Add optional parameter for max. great circle segment length --- scripts-available/CDB_GreatCircle.sql | 4 ++-- test/CDB_GreatCircle.sql | 3 ++- test/CDB_GreatCircle_expect | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_GreatCircle.sql b/scripts-available/CDB_GreatCircle.sql index 8d98dbf..950fae9 100644 --- a/scripts-available/CDB_GreatCircle.sql +++ b/scripts-available/CDB_GreatCircle.sql @@ -1,7 +1,7 @@ -- Great circle point-to-point routes, based on: -- http://blog.cartodb.com/jets-and-datelines/ -- -CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry) +CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry, max_segment_length NUMERIC DEFAULT 100000) RETURNS geometry AS $$ DECLARE line geometry; @@ -11,7 +11,7 @@ BEGIN start_point, end_point )::geography, - 100000 + max_segment_length )::geometry; IF ST_XMax(line) - ST_XMin(line) > 180 THEN diff --git a/test/CDB_GreatCircle.sql b/test/CDB_GreatCircle.sql index c9bfcb6..32b9aa2 100644 --- a/test/CDB_GreatCircle.sql +++ b/test/CDB_GreatCircle.sql @@ -1 +1,2 @@ -select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059))) +select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059))); +select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000)); diff --git a/test/CDB_GreatCircle_expect b/test/CDB_GreatCircle_expect index 0b9f5c7..33dfdf8 100644 --- a/test/CDB_GreatCircle_expect +++ b/test/CDB_GreatCircle_expect @@ -1 +1,2 @@ LINESTRING(4.259 55.858,5.6692453115051 56.0150275120673,7.10720375678704 56.157400475677,8.5718366560563 56.2842986378254,10.0619272412891 56.3949153508462,11.5760785994189 56.4884642014437,13.1127142001617 56.564185934303,14.6700812655504 56.6213555706215,16.2462571744128 56.6592896061102,17.8391590143095 56.6773531596105,19.4465562981665 56.6749669334121,21.0660867567155 56.6516138405427,22.6952750058883 56.6068451534252,24.3315537765309 56.540286032869,25.9722872888145 56.4516403065472,27.6147962622065 56.3406943817481,29.2563839799455 56.207320197769,30.8943627796619 56.0514771479657,32.5260803224591 55.8732129290618,34.1489450028345 55.6726633044968,35.7604499005266 55.4500507979281,37.3581947399686 55.2056823610616,38.9399054089486 54.9399460854786,40.5034506895044 54.6533070499613,42.0468559644411 54.3463024122038,43.5683137754523 54.0195358662507,45.066191217402 53.673671594382,46.5390342525062 53.3094278446298,47.9855691138079 52.9275702630659,49.4047010366934 52.5289051040742,50.7955106088955 52.1142724327331,52.1572480633875 51.6845394219955,53.4893258557794 51.2405938343407,54.7913098701049 50.7833377637432,56.0629095865715 50.3136816997865,57.3039675245588 49.8325389621027,58.5144482465496 49.3408205404538,59.6944271762829 48.8394303639846,60.8440794494795 48.329261012675,61.9636689799149 47.811189874886,63.0535378889196 47.2860757471582,64.1140964137264 46.7547558660385,65.1458133802427 46.2180433565989,66.1492072992903 45.676725078361,67.1248381223566 45.131559846414,68.0732996734468 44.583277003498,68.9952127576034 44.0325753175597,69.8912189338183 43.4801221786776,70.7619749300985 42.9265530691612,71.6081476710291 42.3724712809595,72.4304098829428 41.8184478551838,73.2294362384225 41.2650217194684,74.0059 40.7127) +LINESTRING(4.259 55.858,4.96060044865294 55.9382939511593,5.6692453115051 56.0150275120673,6.38482117645567 56.0880973218335,7.10720375678705 56.157400475677,7.83625773770865 56.2228347173136,8.5718366560563 56.2842986378254,9.31378281572326 56.3416918804739,10.0619272412891 56.3949153508462,10.8160896721679 56.4438714316548,11.5760785994189 56.4884642014437,12.3416913471456 56.528599656387,13.1127142001617 56.5641859343031,13.8889225793161 56.5951335399513,14.6700812655504 56.6213555706215,15.4559446734179 56.6427679409819,16.2462571744128 56.6592896061102,17.0407534700619 56.6708427815999,17.8391590143095 56.6773531596105,18.6411904842936 56.6787501197174,19.4465562981665 56.6749669334121,20.2549571781681 56.6659409611101,21.0660867567155 56.6516138405428,21.8796322228404 56.6319316654367,22.6952750058883 56.6068451534252,23.5126914929996 56.5763098021872,24.3315537765309 56.5402860328691,25.1515304272452 56.4987393199198,25.9722872888145 56.4516403065472,26.7934882889404 56.3989649050969,27.6147962622065 56.3406943817482,28.4358737796488 56.2768154250305,29.2563839799456 56.207320197769,30.0759913971174 56.1322063721813,30.8943627796619 56.0514771479657,31.7111678961496 55.9651412533344,32.5260803224592 55.8732129290618,33.3387782060384 55.7757118957345,34.1489450028345 55.6726633044969,34.9562701828379 55.5640976716962,35.7604499005266 55.4500507979282,36.5611876268714 55.3305636720814,37.3581947399687 55.2056823610617,38.1511910717861 55.0754578859583,38.9399054089486 54.9399460854787,39.7240759459355 54.7992074675415,40.5034506895044 54.6533070499613,41.277787813601 54.5023141912026,42.0468559644411 54.3463024122039,42.8104345158644 54.1853492102971,43.5683137754524 54.0195358662508,44.3202951422663 53.8489472454711,45.066191217402 53.6736715943821,45.8058258688602 53.4938003329924,46.5390342525062 53.3094278446299,47.2656627911282 53.1206512637978,47.985569113808 52.9275702630661,48.6986219579803 52.7302868398744,49.4047010366934 52.5289051040743,50.1036968736777 52.3235310669909,50.7955106088955 52.1142724327332,51.4800537772815 51.9012383924278,52.1572480633875 51.6845394219956,52.8270250346284 51.4642870840378,53.4893258557795 51.2405938343408,54.1441009873167 51.0135728334539,54.791309870105 50.7833377637434,55.4309205988438 50.5500026522693,56.0629095865715 50.3136816997866,56.6872612224038 50.0744891161178,57.3039675245588 49.8325389621028,57.9130277905821 49.5879449982851,58.5144482465496 49.3408205404539,59.1082416968843 49.091278322122,59.6944271762829 48.8394303639847,60.2730296051101 48.5853878503721,60.8440794494795 48.3292610126751,61.40761238711 48.0711590197009,61.9636689799149 47.8111898748862,62.5122943541616 47.5494603202768,63.0535378889195 47.2860757471584,63.5874529134047 47.0211401132109,64.1140964137264 46.7547558660387,64.6335287494427 46.4870238729191,65.1458133802426 46.2180433565991,65.6510166029904 45.9479118369597,66.1492072992903 45.6767250783612,66.6404566936622 45.4045770424768,67.1248381223566 45.1315598464143,67.6024268127789 44.8577637259253,68.0732996734467 44.5832770034983,68.5375350943572 44.3081860611283,68.9952127576034 44.0325753175599,69.4464134580461 43.7565272097974,69.8912189338183 43.4801221786779,70.3297117064144 43.2034386583077,70.7619749300985 42.9265530691615,71.1880922503468 42.6495398146493,71.6081476710291 42.3724712809597,72.0222254300221 42.0954178399905,72.4304098829428 41.8184478551841,72.8327853946822 41.5416276900883,73.2294362384225 41.2650217194687,73.6204465018146 40.9886923428039,74.0059 40.7127)