From 1ba7299fe4f049b8000317cff1a1d851f3678f23 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:02:35 -0400 Subject: [PATCH 01/13] adds obs_getmeasurebyid --- src/pg/sql/41_observatory_augmentation.sql | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index a8c42b2..75121b3 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -380,6 +380,50 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureById( + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL +) +RETURNS NUMERIC +AS $$ +DECLARE + target_table TEXT; + colname TEXT; + measure_val NUMERIC; + data_geoid_colname TEXT; +BEGIN + -- TODO look at how timespans are decided in other functions + SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table + FROM cdb_observatory._OBS_GetColumnData(boundary_id, Array[measure_id], time_span) As x; + + EXECUTE + format('SELECT ct.colname + FROM observatory.obs_column_to_column c2c, + observatory.obs_column_table ct, + observatory.obs_table t + WHERE c2c.reltype = ''geom_ref'' + AND ct.column_id = c2c.source_id + AND ct.table_id = t.id + AND t.tablename = %L' + , target_table) + INTO data_geoid_colname; + + EXECUTE format( + 'SELECT %I + FROM observatory.%I + WHERE %I.%I = %L' + colname, + target_table, + target_table, data_geoid_colname, measure_id) + INTO measure_val; + + RETURN measure_val; + +END; +$$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory( geom geometry(Geometry, 4326), category_id TEXT, From 3f77a384c75800096cb16a47c112c4e4bc6f5f90 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:06:53 -0400 Subject: [PATCH 02/13] adds missing comma --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 75121b3..b80f807 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -413,7 +413,7 @@ BEGIN EXECUTE format( 'SELECT %I FROM observatory.%I - WHERE %I.%I = %L' + WHERE %I.%I = %L', colname, target_table, target_table, data_geoid_colname, measure_id) From 2e312464aa8cbaa7e236bc567e95647756a3e613 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:22:39 -0400 Subject: [PATCH 03/13] fixes geom_ref in where condition --- src/pg/sql/41_observatory_augmentation.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index b80f807..349ca69 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -393,6 +393,7 @@ DECLARE colname TEXT; measure_val NUMERIC; data_geoid_colname TEXT; + test_query TEXT; BEGIN -- TODO look at how timespans are decided in other functions SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table @@ -410,13 +411,18 @@ BEGIN , target_table) INTO data_geoid_colname; - EXECUTE format( + RAISE NOTICE 'target_table %, colname %', target_table, colname; + + test_query := format( 'SELECT %I FROM observatory.%I WHERE %I.%I = %L', colname, target_table, - target_table, data_geoid_colname, measure_id) + target_table, data_geoid_colname, geom_ref); + RAISE NOTICE 'test_query: %', test_query; + + EXECUTE test_query INTO measure_val; RETURN measure_val; From 94756a2377174f878b9a80dc7640fb8c2c926485 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:27:41 -0400 Subject: [PATCH 04/13] minor formatting changes --- src/pg/sql/41_observatory_augmentation.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 349ca69..ec721f5 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -411,18 +411,15 @@ BEGIN , target_table) INTO data_geoid_colname; - RAISE NOTICE 'target_table %, colname %', target_table, colname; + RAISE DEBUG 'target_table %, colname %', target_table, colname; - test_query := format( + EXECUTE format( 'SELECT %I FROM observatory.%I WHERE %I.%I = %L', colname, target_table, - target_table, data_geoid_colname, geom_ref); - RAISE NOTICE 'test_query: %', test_query; - - EXECUTE test_query + target_table, data_geoid_colname, geom_ref) INTO measure_val; RETURN measure_val; From afb548c75d70432897681a78c53ccf2fcecca99d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 12:02:32 -0400 Subject: [PATCH 05/13] adds tests --- .../41_observatory_augmentation_test.out | 9 ++++++++ .../sql/41_observatory_augmentation_test.sql | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 0094351..959549c 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -76,3 +76,12 @@ t obs_getuscensuscategory_polygon t (1 row) +obs_getmeasurebyid_cartodb_census_tract +t +(1 row) +obs_getmeasurebyid_cartodb_block_group +t +(1 row) +obs_getmeasurebyid_nulls +t +(1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 9a14047..3370e78 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -204,4 +204,27 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory( cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Low income, mix of minorities' As OBS_GetUSCensusCategory_polygon; +-- OBS_GetMeasureById tests +SELECT (cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + 'us.census.tiger.census_tract', + '2010 - 2014' +) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; + +SELECT (cdb_observatory.OBS_GetMeasureById( + '360470485002', + 'us.census.acs.B01003001', + 'us.census.tiger.block_group', + '2010 - 2014' +) - 1900) / 1900 < 0.0001 As OBS_GetMeasureById_cartodb_block_group; + +-- geom ref / boundary mismatch +SELECT cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + 'us.census.tiger.block_group', + '2010 - 2014' +) IS NULL As OBS_GetMeasureById_nulls; + \i test/fixtures/drop_fixtures.sql From b5267c14bf5d9b3af6e3bdac0351ef131f70c9dc Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:15:19 -0400 Subject: [PATCH 06/13] removes unneeded todo --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ec721f5..e355950 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -395,7 +395,7 @@ DECLARE data_geoid_colname TEXT; test_query TEXT; BEGIN - -- TODO look at how timespans are decided in other functions + SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table FROM cdb_observatory._OBS_GetColumnData(boundary_id, Array[measure_id], time_span) As x; From f9ecf1595c575f34bd1fda3a5dfe54dccb1c8090 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:28:21 -0400 Subject: [PATCH 07/13] adds docs for obs_getmeasurebyid --- doc/measures_functions.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/doc/measures_functions.md b/doc/measures_functions.md index a07ad93..2403e1b 100644 --- a/doc/measures_functions.md +++ b/doc/measures_functions.md @@ -134,6 +134,40 @@ SET household_count = OBS_GetMeasure(the_geom, 'us.census.acs.B11001001') * If an unrecognized normalization type is input, raise an error: `'Only valid inputs for "normalize" are "area" (default) and "denominator".` +## OBS_GetMeasureById(geom_ref text, measure_id text, boundary_id text) + +The ```OBS_GetMeasureById(geom_ref, measure_id, boundary_id)``` function returns any Data Observatory measure that corresponds to the boundary in ```boundary_id``` that has a geometry reference of ```geom_ref```. + +#### Arguments + +Name |Description +--- | --- +geom_ref | a geometry reference (e.g., a US Census geoid) +measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf)) +boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract') +time_span (optional) | time span of interest (e.g., 2010 - 2014) + +#### Returns + +A NUMERIC value + +Key | Description +--- | --- +value | the raw measure associated with `geom_ref` + +#### Example + +Add a measure to an empty column based on county geoids in your table + +```SQL +UPDATE tablename +SET household_count = OBS_GetMeasureById(geoid_column, 'us.census.acs.B11001001', 'us.census.tiger.county') +``` + +#### Errors + +* Returns `NULL` if there is a mismatch between the geometry reference and the boundary id such as using the geoid of a county with the boundary of block groups + ## OBS_GetCategory(point geometry, category_id text) The ```OBS_GetCategory(point, category_id)``` function returns any Data Observatory Category value at a point location. The Categories available are currently limited to Segmentation categories. See the Segmentation section of the [Catalog](https://cartodb.github.io/bigmetadata/observatory.pdf) for more detail. From 350228283592189a2f3a7f60c05ed482498594a2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:30:22 -0400 Subject: [PATCH 08/13] adds more description for timespan --- doc/measures_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/measures_functions.md b/doc/measures_functions.md index 2403e1b..be7e9f8 100644 --- a/doc/measures_functions.md +++ b/doc/measures_functions.md @@ -145,7 +145,7 @@ Name |Description geom_ref | a geometry reference (e.g., a US Census geoid) measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf)) boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract') -time_span (optional) | time span of interest (e.g., 2010 - 2014) +time_span (optional) | time span of interest (e.g., 2010 - 2014). If `NULL` is passed, the measure from the most recent data will be used. #### Returns From 49d584822c07d61d9a073dbb24f594612cb275bc Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 13:28:56 -0400 Subject: [PATCH 09/13] adds another test on different types of inputs --- src/pg/test/expected/41_observatory_augmentation_test.out | 3 +++ src/pg/test/sql/41_observatory_augmentation_test.sql | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 959549c..8311660 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -79,6 +79,9 @@ t obs_getmeasurebyid_cartodb_census_tract t (1 row) +obs_getmeasurebyid_null_boundary_null_timespan +t +(1 row) obs_getmeasurebyid_cartodb_block_group t (1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 3370e78..49513c6 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -212,6 +212,14 @@ SELECT (cdb_observatory.OBS_GetMeasureById( '2010 - 2014' ) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; +-- OBS_GetMeasureById tests +SELECT cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + NULL, + NULL +) IS NULL As OBS_GetMeasureById_null_boundary_null_timespan; + SELECT (cdb_observatory.OBS_GetMeasureById( '360470485002', 'us.census.acs.B01003001', From 076e285ed7d9be84d455fe4f48addadd8cbdb633 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 13:37:02 -0400 Subject: [PATCH 10/13] adds notes on test calls --- src/pg/test/sql/41_observatory_augmentation_test.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 92eb224..8a3f023 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -205,6 +205,7 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory( -- OBS_GetMeasureById tests +-- typical query SELECT (cdb_observatory.OBS_GetMeasureById( '36047048500', 'us.census.acs.B01003001', @@ -212,7 +213,7 @@ SELECT (cdb_observatory.OBS_GetMeasureById( '2010 - 2014' ) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; --- OBS_GetMeasureById tests +-- no boundary_id should give null SELECT cdb_observatory.OBS_GetMeasureById( '36047048500', 'us.census.acs.B01003001', @@ -220,6 +221,7 @@ SELECT cdb_observatory.OBS_GetMeasureById( NULL ) IS NULL As OBS_GetMeasureById_null_boundary_null_timespan; +-- query at block_group level SELECT (cdb_observatory.OBS_GetMeasureById( '360470485002', 'us.census.acs.B01003001', From 8fbb9ebbccf2053def1133e28538e9849e76703c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 13:48:37 -0400 Subject: [PATCH 11/13] updating version to 0.0.5 --- src/pg/observatory.control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/observatory.control b/src/pg/observatory.control index 279edf1..1e13336 100644 --- a/src/pg/observatory.control +++ b/src/pg/observatory.control @@ -1,5 +1,5 @@ comment = 'CartoDB Observatory backend extension' -default_version = '0.0.4' +default_version = '0.0.5' requires = 'postgis' superuser = true schema = cdb_observatory From e960f3e0970605866d78e201369b34f65e012ad2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 14:13:12 -0400 Subject: [PATCH 12/13] backfilling news --- NEWS.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7d01917..a9d6140 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,28 @@ -0.0.1 (open date) +0.0.5 (5/27/2016) +----- +* Adds new function `OBS_GetMeasureById` ([#96](https://github.com/CartoDB/observatory-extension/pull/96)) + +0.0.4 (5/25/2016) +----- +* Updates queries involving US Census measure tags to be more generic ([#95](https://github.com/CartoDB/observatory-extension/pull/95)) +* Fixes tests which relied on an erroneous subset of block groups ([#95](https://github.com/CartoDB/observatory-extension/pull/95)) + +0.0.3 (5/24/2016) +----- +* Generalizes internal queries to properly pull from multiple named geometry references +* Adds tests for Who's on First boundaries +* Improves automatic fixtures testing script + +0.0.2 (5/19/2016) +----- +* Adds Data Observatory exploration functions +* Adds Data Observatory boundary functions +* Adds Data Observatory measure functions +* Adds script to generate fixtures for tests +* Adds script for the automatic testing of metadata +* Adds full documentation for all included functions +* removes `cartodb` extension dependency + +0.0.1 (5/19/2016) ------------------ -* First iteration of `OBS_GetDemographicSnapshot(location Geometry(Point,4326))`; +* First iteration of `OBS_GetDemographicSnapshot(location Geometry(Point,4326))` From 5e99b6032961145d728f139bc1acd6a6b5c33567 Mon Sep 17 00:00:00 2001 From: csobier Date: Mon, 30 May 2016 11:05:50 -0400 Subject: [PATCH 13/13] updated license INE description --- doc/license.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/license.md b/doc/license.md index d799bfb..05bcb5c 100644 --- a/doc/license.md +++ b/doc/license.md @@ -14,5 +14,5 @@ Natural Earth | [http://www.naturalearthdata.com/about/terms-of-use/](http://w Quattroshapes | [https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md](https://github.com/foursquare/quattroshapes/blob/master/LICENSE.md) Zetashapes | [http://zetashapes.com/license](http://zetashapes.com/license) Spielman & Singleton | [https://www.openicpsr.org/repoEntity/show/41329](https://www.openicpsr.org/repoEntity/show/41329) -Instituto Nacional de Estadistica | [http://www.ine.es/ss/Satellite?L=0&c=Page&cid=1254735849170&p=1254735849170&pagename=Ayuda%2FINELayout](http://www.ine.es/ss/Satellite?L=0&c=Page&cid=1254735849170&p=1254735849170&pagename=Ayuda%2FINELayout) +El Instituto Nacional de Estadística (INE) | The National Statistics Institute (INE) of Spain includes data from multiple sources. If you are re-using their data, they explicitly require that you reference them accordingly

[http://www.ine.es/ss/Satellite?L=0&c=Page&cid=1254735849170&p=1254735849170&pagename=Ayuda%2FINELayout](http://www.ine.es/ss/Satellite?L=0&c=Page&cid=1254735849170&p=1254735849170&pagename=Ayuda%2FINELayout)