From 9d3bb4087727272bf0e06a5f8f24d0961025b16c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 11 May 2016 11:39:23 -0400 Subject: [PATCH 1/6] scratch work to get _obs_columndata to return better defaults --- src/pg/sql/40_observatory_utility.sql | 11 +++++++++-- src/pg/sql/41_observatory_augmentation.sql | 14 +++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 38ba4a6..c3fc1f9 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -52,10 +52,17 @@ BEGIN EXECUTE ' WITH geomref AS ( SELECT t.table_id id - FROM observatory.OBS_column_to_column c2c, observatory.OBS_column_table t + FROM observatory.OBS_column_to_column c2c, + observatory.OBS_column c, + observatory.OBS_column_table geom_ct, + observatory.OBS_column_table data_ct, + observatory.OBS_table t WHERE c2c.reltype = ''geom_ref'' + AND c.column_id = c2c.target_id AND c2c.target_id = $1 - AND c2c.source_id = t.column_id + AND c2c.source_id = ct.column_id + ORDER BY weight DESC + LIMIT 1 ), column_ids as ( select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ef78434..2310b49 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -219,6 +219,7 @@ BEGIN -- TODO we're assuming our geom_table has only one geom_ref column -- we *really* should pass in both geom_table_name and boundary_id + -- TODO tablename should not be passed here (use boundary_id) EXECUTE format('SELECT ct.colname FROM observatory.obs_column_to_column c2c, @@ -284,10 +285,11 @@ BEGIN query := query || format(' ]::numeric[] FROM observatory.%I - WHERE %I.geoid = %L + WHERE %I.%I = %L ', ((data_table_info)[1])->>'tablename', ((data_table_info)[1])->>'tablename', + geoid_colname, geoid ); @@ -332,16 +334,6 @@ DECLARE vals NUMERIC[]; BEGIN - IF boundary_id IS NULL THEN - -- TODO we should determine best boundary for this geom - boundary_id := 'us.census.tiger.block_group'; - END IF; - - IF time_span IS NULL THEN - -- TODO we should determine latest timespan for this measure - time_span := '2010 - 2014'; - END IF; - IF normalize ILIKE 'area' THEN measure_ids := ARRAY[measure_id]; ELSIF normalize ILIKE 'denominator' THEN From b2b34bfe053eab159d94acec939a7e420189bf31 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 11 May 2016 17:55:52 -0400 Subject: [PATCH 2/6] default to latest if unspecified timespan/boundary id --- src/pg/sql/40_observatory_utility.sql | 36 +++++++++++++++---- .../41_observatory_augmentation_test.out | 3 ++ .../sql/41_observatory_augmentation_test.sql | 6 ++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index c3fc1f9..bf01fe0 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -48,21 +48,43 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( RETURNS SETOF JSON AS $$ BEGIN + + -- figure out highest-weight geometry_id/timespan pair for the first data column + -- TODO this should be done for each data column separately + IF geometry_id IS NULL OR timespan IS NULL THEN + EXECUTE ' + SELECT data_t.timespan timespan, geom_c.id boundary_id + FROM observatory.obs_table data_t, + observatory.obs_column_table data_ct, + observatory.obs_column data_c, + observatory.obs_column_table geoid_ct, + observatory.obs_column_to_column c2c, + observatory.obs_column geom_c + WHERE data_c.id = $2 + AND data_ct.column_id = data_c.id + AND data_ct.table_id = data_t.id + AND geoid_ct.table_id = data_t.id + AND geoid_ct.column_id = c2c.source_id + AND c2c.reltype = ''geom_ref'' + AND geom_c.id = c2c.target_id + AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END + AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END + ORDER BY geom_c.weight DESC, + data_t.timespan DESC + LIMIT 1; + ' INTO timespan, geometry_id + USING geometry_id, column_ids[1], timespan; + END IF; + RETURN QUERY EXECUTE ' WITH geomref AS ( SELECT t.table_id id FROM observatory.OBS_column_to_column c2c, - observatory.OBS_column c, - observatory.OBS_column_table geom_ct, - observatory.OBS_column_table data_ct, - observatory.OBS_table t + observatory.OBS_column_table ct WHERE c2c.reltype = ''geom_ref'' - AND c.column_id = c2c.target_id AND c2c.target_id = $1 AND c2c.source_id = ct.column_id - ORDER BY weight DESC - LIMIT 1 ), column_ids as ( select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index e1ea9d7..0094351 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -37,6 +37,9 @@ t obs_getmeasure_zhvi_point_test t (1 row) +obs_getmeasure_zhvi_point_default_latest_test +t +(1 row) obs_getmeasure_total_pop_point_test t (1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index a874bbd..0899926 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -139,6 +139,12 @@ SELECT abs(OBS_GetMeasure_zhvi_point - 583600) / 583600 < 0.001 AS OBS_GetMeasur 'us.zillow.AllHomes_Zhvi', 'area', 'us.census.tiger.zcta5', '2014-01' ) As t(OBS_GetMeasure_zhvi_point); +-- Point-based OBS_GetMeasure with zillow default to latest +SELECT abs(OBS_GetMeasure_zhvi_point_default_latest - 583600) / 583600 < 0.001 AS OBS_GetMeasure_zhvi_point_default_latest_test FROM cdb_observatory.OBS_GetMeasure( + ST_SetSRID(ST_Point( -73.94602417945862, 40.6768220087458), 4326), + 'us.zillow.AllHomes_Zhvi' +) As t(OBS_GetMeasure_zhvi_point_default_latest); + -- Point-based OBS_GetMeasure, default normalization (area) -- is result within 0.1% of expected SELECT abs(OBS_GetMeasure_total_pop_point - 10923.093200390833950) / 10923.093200390833950 < 0.001 As OBS_GetMeasure_total_pop_point_test FROM From d1c2598167f3c3490ecdc056b665bcbc1d341f31 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 11 May 2016 17:57:46 -0400 Subject: [PATCH 3/6] fix bug with table alias --- src/pg/sql/40_observatory_utility.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index bf01fe0..42fb886 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -79,7 +79,7 @@ BEGIN RETURN QUERY EXECUTE ' WITH geomref AS ( - SELECT t.table_id id + SELECT ct.table_id id FROM observatory.OBS_column_to_column c2c, observatory.OBS_column_table ct WHERE c2c.reltype = ''geom_ref'' From ccf61c35833d628c3dc9bd23eadd36b34e7b6491 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 11 May 2016 17:59:58 -0400 Subject: [PATCH 4/6] array typing issues --- src/pg/sql/40_observatory_utility.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 42fb886..f132d85 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -71,9 +71,9 @@ BEGIN AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END ORDER BY geom_c.weight DESC, data_t.timespan DESC - LIMIT 1; + LIMIT 1 ' INTO timespan, geometry_id - USING geometry_id, column_ids[1], timespan; + USING geometry_id, (column_ids)[1], timespan; END IF; RETURN QUERY From 506f9ead302fa8c7111de45d5ced20f54f1fcfd7 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 11 May 2016 18:17:48 -0400 Subject: [PATCH 5/6] first-pass defaults to latest timespan/boundary_id --- src/pg/sql/40_observatory_utility.sql | 3 ++- src/pg/sql/41_observatory_augmentation.sql | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index f132d85..6ed44b2 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -95,7 +95,8 @@ BEGIN aggregate, name, type, - c.description + c.description, + $1 AS boundary_id FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t WHERE column_ids.column_id = c.id AND c.id = ct.column_id diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 414b10e..768626f 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -155,20 +155,25 @@ DECLARE data_table_info json[]; BEGIN - geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level); - - IF geom_table_name IS NULL - THEN - RAISE NOTICE 'Point % is outside of the data region', ST_AsText(geom); - RETURN QUERY SELECT '{}'::text[], '{}'::NUMERIC[]; - END IF; - EXECUTE 'SELECT array_agg(_obs_getcolumndata) FROM cdb_observatory._OBS_GetColumnData($1, $2, $3);' INTO data_table_info USING geometry_level, column_ids, time_span; + IF geometry_level IS NULL THEN + geometry_level = data_table_info[1]->>'boundary_id'; + END IF; + + geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level); + + IF geom_table_name IS NULL + THEN + RAISE NOTICE 'Point % is outside of the data region', ST_AsText(geom); + -- TODO this should return JSON + RETURN QUERY SELECT '{}'::text[], '{}'::NUMERIC[]; + END IF; + IF data_table_info IS NULL THEN RAISE NOTICE 'Cannot find data table for boundary ID %, column_ids %, and time_span %', geometry_level, column_ids, time_span; END IF; From 3f1b1cfcb929fa6bf0eb5a0063a3f96023118005 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 12 May 2016 09:39:44 -0400 Subject: [PATCH 6/6] update tests to work with latest zillow --- src/pg/test/fixtures/drop_fixtures.sql | 1 + src/pg/test/fixtures/load_fixtures.sql | 256 +++++++++++++++++- .../test/sql/40_observatory_utility_test.sql | 4 +- .../sql/41_observatory_augmentation_test.sql | 6 +- 4 files changed, 261 insertions(+), 6 deletions(-) diff --git a/src/pg/test/fixtures/drop_fixtures.sql b/src/pg/test/fixtures/drop_fixtures.sql index 74e0695..e7681ae 100644 --- a/src/pg/test/fixtures/drop_fixtures.sql +++ b/src/pg/test/fixtures/drop_fixtures.sql @@ -10,6 +10,7 @@ DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d; DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd; DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7; DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81; +DROP TABLE IF EXISTS observatory.obs_8764a6b439a4f8714f54d4b3a157bc5e36519066; DROP TABLE IF EXISTS observatory.obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb; DROP TABLE IF EXISTS observatory.obs_d34555209878e8c4b37cf0b2b3d072ff129ec470; DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13; diff --git a/src/pg/test/fixtures/load_fixtures.sql b/src/pg/test/fixtures/load_fixtures.sql index 58a6a93..6484bd0 100644 --- a/src/pg/test/fixtures/load_fixtures.sql +++ b/src/pg/test/fixtures/load_fixtures.sql @@ -12361,7 +12361,261 @@ cartodb_id,the_geom,the_geom_webmercator,geoid,aland,awater \. CREATE SCHEMA IF NOT EXISTS "observatory"; - ALTER TABLE "obs_1babf5a26a1ecda5fb74963e88408f71d0364b81" SET SCHEMA "observatory";DROP TABLE IF EXISTS "observatory"."obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb"; + ALTER TABLE "obs_1babf5a26a1ecda5fb74963e88408f71d0364b81" SET SCHEMA "observatory";DROP TABLE IF EXISTS "observatory"."obs_8764a6b439a4f8714f54d4b3a157bc5e36519066"; + + CREATE TABLE "obs_8764a6b439a4f8714f54d4b3a157bc5e36519066" ( + cartodb_id NUMERIC, the_geom GEOMETRY, the_geom_webmercator GEOMETRY, region_name TEXT, singlefamilyresidence_zhvi NUMERIC, allhomes_zhvi NUMERIC, allhomes_medianvaluepersqft NUMERIC, allhomes_medianrentalpricepersqft NUMERIC, sfr_medianrentalpricepersqft NUMERIC, singlefamilyresidencerental_zri NUMERIC, allhomesplusmultifamily_zri NUMERIC + ); + +COPY "obs_8764a6b439a4f8714f54d4b3a157bc5e36519066" FROM stdin WITH CSV HEADER; +cartodb_id,the_geom,the_geom_webmercator,region_name,singlefamilyresidence_zhvi,allhomes_zhvi,allhomes_medianvaluepersqft,allhomes_medianrentalpricepersqft,sfr_medianrentalpricepersqft,singlefamilyresidencerental_zri,allhomesplusmultifamily_zri +16,,,11703,307900,307900,,,,, +17,,,11726,251400,251400,,,,, +46,,,11940,355300,359800,,,,, +67,,,11955,388100,380800,,,,, +68,,,11980,283200,249400,,,,, +84,,,11796,293300,293300,,,,, +87,,,11719,334700,334700,,,,, +97,,,11718,458000,458000,,,,, +107,,,11724,1283500,1283500,,,,, +141,,,11939,659500,659400,,,,, +156,,,11948,627800,627800,,,,, +159,,,11957,828700,828700,,,,, +163,,,11958,771500,771500,,,,, +164,,,11739,531700,531700,,,,, +207,,,11757,286600,286600,156,,,, +225,,,11704,292600,292600,119,,,, +252,,,11901,318800,322300,231,,,, +297,,,11701,272600,272600,191,,,, +323,,,11967,207400,207600,125,,,, +347,,,11946,467800,466600,343,,,, +404,,,11961,239900,217000,173,,,, +430,,,11702,376100,376100,209,,,, +439,,,11949,378800,370400,170,,,, +463,,,11951,167800,167800,135,,,, +468,,,11950,174700,174700,113,,,, +469,,,11778,250200,250200,165,,,, +471,,,11730,358100,358100,213,,,, +521,,,11713,265000,265000,177,,,, +536,,,11769,350800,351700,169,,,, +541,,,11954,1223600,1167200,689,,,, +559,,,11971,583000,585300,306,,,, +575,,,11792,390100,390100,208,,,, +589,,,11933,345800,350200,216,,,, +602,,,11934,325700,327000,178,,,, +625,,,11942,577400,577400,358,,,, +640,,,11944,492100,496600,254,,,, +643,,,11952,504900,504900,300,,,, +717,,,11786,336000,336000,162,,,, +768,,,11935,733700,733700,464,,,, +770,,,11976,3406700,3406700,1074,,,, +815,,,11977,799900,799900,386,,,, +919,,,11932,2323700,2323700,887,,,, +924,,,11509,886800,873900,414,,,, +1070,,,11960,1235800,1235800,478,,,, +1215,,,11959,1583700,1583700,612,16.1764705882353,16.1764705882353,, +1216,,,11219,922300,,,,,2279,2251 +1228,,,11777,477000,484000,217,,,,3276 +1234,,,11561,513800,518200,333,3.18181818181818,,,2794 +1240,,,11102,894800,,,,,2488,2444 +1243,,,11104,888200,,,,,2488,2348 +1263,,,11216,1437100,972900,,,,2510,2519 +1276,,,11721,480900,480900,,,,3104,3103 +1295,,,11729,299000,299000,,,,2785,2780 +1315,,,11370,662600,588800,,,,2354,2346 +1343,,,11798,210400,210400,,,,2818,2812 +1359,,,11740,438000,438000,,,,3068,3066 +1362,,,11752,314000,314000,,,,2788,2781 +1435,,,11715,329900,331000,,,,2871,2840 +1532,,,11749,271300,272100,,,,2851,2825 +1650,,,11209,970500,,556,,,2378,2242 +1651,,,11230,954500,,511,,,2327,2222 +1652,,,11372,801000,,375,,,2400,2243 +1653,,,11233,781500,,509,,,2409,2347 +1654,,,11218,1062400,,601,,,2428,2301 +1655,,,11238,2293400,,930,,,2784,2776 +1656,,,11223,871500,,535,,,2317,2216 +1657,,,11220,816600,,595,,,2306,2240 +1662,,,11360,925100,,480,,,2432,2277 +1689,,,11226,846800,573800,482,,,2316,2111 +1694,,,11212,316400,317600,270,,,2131,2098 +1695,,,11214,732700,550900,462,,,2213,2178 +1699,,,11377,669300,406500,459,,,2368,2365 +1700,,,11234,508000,491000,364,,,2163,2119 +1705,,,11207,353800,331100,300,,,2222,2275 +1706,,,11385,558500,545300,386,,,2296,2303 +1710,,,11215,1935100,1061600,998,,,2801,2803 +1712,,,11368,649700,374600,494,,,2353,2342 +1714,,,11221,1018900,886200,525,,,2551,2516 +1718,,,11373,723100,316200,447,,,2368,2275 +1726,,,11236,399400,380100,307,,,2139,2073 +1759,,,11208,377600,371900,293,,,2298,2297 +1770,,,11204,813600,748100,470,,,2254,2223 +1780,,,11225,1393800,926000,601,,,2442,2416 +1781,,,11213,1070600,696400,456,,,2401,2358 +1785,,,11203,420700,419700,279,,,2059,2046 +1822,,,11746,426800,426800,202,,,3200,3189 +1846,,,11210,700900,635600,419,,,2223,2165 +1873,,,11434,390600,390600,305,,,2113,2106 +1894,,,11224,434600,389100,365,,,2223,2215 +1912,,,11691,427600,396300,264,,,2108,2063 +1966,,,11758,436300,435900,276,,,2708,2694 +1968,,,11706,265200,265300,176,,,2776,2756 +2011,,,11106,753500,287700,596,,,2511,2430 +2048,,,11432,732100,619400,421,,,2370,2251 +2107,,,11772,250000,251100,162,,,2697,2673 +2208,,,11435,456400,375300,347,,,2165,2145 +2234,,,11550,285700,286200,204,,,2572,2563 +2263,,,11205,,712400,595,,,2814,2753 +2286,,,11743,540600,540000,244,,,3345,3326 +2371,,,11217,2962900,1121600,1033,,,2924,2859 +2374,,,11365,779700,749600,491,,,2403,2363 +2413,,,11367,704300,472800,409,,,2366,2233 +2420,,,11103,854300,689500,617,,,2485,2468 +2541,,,11231,2691200,1093000,1012,,,2941,2914 +2568,,,11590,401600,411400,257,,,2720,2719 +2583,,,11520,315700,318600,205,,,2596,2584 +2669,,,11040,583300,582900,375,,,2952,2934 +2682,,,11756,360600,360300,247,,,2585,2582 +2732,,,11357,837400,762500,528,,,2357,2303 +2832,,,11779,296300,296200,176,,,2667,2658 +2863,,,11358,789700,766100,480,,,2397,2354 +2883,,,11364,815000,634400,519,,,2408,2289 +2965,,,11554,427300,428700,291,,,2699,2693 +3054,,,11717,239400,239400,126,,,2736,2725 +3087,,,11801,412900,412900,293,,,2657,2647 +3106,,,11787,421100,421000,244,,,2885,2882 +3195,,,11580,383300,380900,274,,,2572,2560 +3199,,,11566,493000,493100,284,,,2854,2844 +3241,,,11050,841800,844800,452,,,3968,3648 +3304,,,11710,437200,436300,276,,,2716,2702 +3411,,,11510,367900,367400,232,,,2621,2614 +3415,,,11228,841700,818700,535,,,2275,2237 +3425,,,11727,267700,260400,150,,,2655,2606 +3468,,,11379,657800,648700,445,,,2335,2327 +3520,,,11735,375600,374100,261,,,2626,2610 +3594,,,11793,440500,437600,283,,,2711,2701 +3610,,,11003,356700,356700,269,,,2556,2540 +3640,,,11413,418200,417900,292,,,2119,2115 +3688,,,11572,431200,428200,257,,,2776,2748 +3778,,,11420,433900,433500,338,,,2106,2099 +3841,,,11570,666200,664100,327,,,3226,3100 +3851,,,11414,628100,484400,297,,,2202,2124 +3855,,,11731,397100,397100,186,,,3012,3008 +3877,,,11722,227800,228700,127,,,2732,2716 +3944,,,11530,806100,804600,416,,,3917,3780 +3983,,,11542,510200,508900,295,,,2929,2854 +3987,,,11803,542400,537400,329,,,3051,3025 +4006,,,11741,318900,318900,167,,,2738,2727 +4009,,,11725,415200,415200,267,,,2927,2926 +4041,,,11361,767400,716800,516,,,2398,2337 +4095,,,11412,403700,403600,275,,,2188,2180 +4125,,,11763,260500,256700,153,,,2693,2624 +4135,,,11378,619600,607200,428,,,2325,2332 +4170,,,11421,448600,441500,327,,,2191,2177 +4303,,,11720,279400,279400,159,,,2603,2602 +4304,,,11419,443800,443800,328,,,2141,2132 +4409,,,11433,386300,384400,304,,,2161,2161 +4438,,,11418,505500,464500,328,,,2253,2206 +4451,,,11415,879600,185400,392,,,2336,2172 +4593,,,11001,531100,531200,340,,,2811,2743 +4646,,,11776,290700,289200,167,,,2702,2672 +4660,,,11791,660600,657500,361,,,3455,3436 +4680,,,11021,1180500,1157300,466,,,5177,3751 +4702,,,11795,382700,382700,190,,,3066,3051 +4741,,,11563,427100,422500,271,,,2665,2604 +4749,,,11768,572200,572200,291,,,3388,3373 +4759,,,11423,486700,470600,321,,,2291,2252 +4799,,,11714,416000,416600,285,,,2656,2641 +4845,,,11369,600100,570800,376,,,2311,2320 +4847,,,11747,550900,551100,232,,,3406,3439 +4878,,,11784,249000,249000,138,,,2574,2569 +4905,,,11501,471400,474200,326,,,2790,2743 +4948,,,11422,416300,411300,304,,,2112,2102 +4958,,,11552,412500,412500,267,,,2657,2643 +4959,,,11694,765500,678700,366,,,2275,2164 +4975,,,11010,441200,441200,315,,,2694,2664 +4988,,,11762,431500,431000,286,,,2713,2702 +5084,,,11232,856500,492100,620,,,2374,2323 +5101,,,11417,456500,450400,342,,,2136,2127 +5158,,,11427,524500,482000,372,,,2317,2285 +5203,,,11783,420600,420600,271,,,2670,2661 +5212,,,11692,302400,301900,234,,,2065,2052 +5250,,,11356,655100,611300,449,,,2280,2286 +5337,,,11581,421500,420900,265,,,2709,2676 +5426,,,11754,390000,393600,247,,,2827,2813 +5574,,,11733,449500,451200,223,,,3030,2988 +5577,,,11030,1516100,1452500,573,,,6923,6436 +5681,,,11553,314300,314300,241,,,2602,2592 +5724,,,11782,363600,364100,201,,,2903,2890 +5784,,,11429,430600,430500,298,,,2224,2217 +5799,,,11788,421100,421100,239,,,3043,3033 +5870,,,11780,448000,445900,252,,,3041,3031 +5875,,,11362,895100,735500,506,,,2415,2299 +5933,,,11411,421800,421800,314,,,2147,2142 +5980,,,11426,537000,520500,388,,,2327,2303 +6052,,,11428,458700,458700,337,,,2275,2266 +6057,,,11953,246200,229700,158,,,3078,2915 +6185,,,11738,288000,288000,153,,,2590,2589 +6281,,,11751,320600,320600,187,,,2842,2803 +6391,,,11790,375100,373500,196,,,2855,2841 +6395,,,11004,592000,544300,476,,,2363,2259 +6406,,,11693,355700,328000,277,,,2054,2039 +6419,,,11767,392600,392600,224,,,2840,2838 +6555,,,11742,316200,315900,194,,,2677,2649 +6579,,,11764,359500,366600,183,,,3224,3158 +6596,,,11436,384100,384100,339,,,2096,2092 +6641,,,11416,455200,452000,338,,,2164,2144 +6657,,,11576,1114700,1006700,392,,,5042,4534 +6763,,,11766,363300,379100,185,,,2982,2962 +6767,,,11755,350600,360300,178,,,2866,2858 +6809,,,11753,759800,755300,368,,,3851,3899 +6823,,,11577,803300,817600,370,,,3890,3812 +6845,,,11598,678700,678900,279,,,3386,3353 +6861,,,11366,793500,787200,517,,,2406,2383 +6922,,,11575,273500,273500,201,,,2562,2557 +6955,,,11545,895200,895500,393,,,4316,4195 +7005,,,11518,439200,439300,273,,,2729,2661 +7093,,,11771,835700,842200,352,,,3838,3447 +7113,,,11716,363400,363400,231,,,2937,2920 +7306,,,11596,589400,591900,368,,,3004,2973 +7553,,,11797,1052400,895400,330,,,4742,4327 +7614,,,11023,1024200,1026300,449,,,4656,4512 +7656,,,11565,439100,439100,276,,,2691,2686 +7829,,,11705,375500,376700,206,,,3011,2988 +7857,,,11789,235600,235600,168,,,2890,2885 +7860,,,11557,592400,584400,277,,,3174,3041 +7917,,,11559,1114400,1053000,347,,,5030,4313 +7951,,,11558,393900,398800,246,,,2659,2625 +8101,,,11363,1006400,948800,552,,,2419,2392 +8150,,,11516,592500,590200,279,,,3211,3099 +8172,,,11709,546800,546800,324,,,2995,2968 +8177,,,11560,758200,758200,382,,,3614,3492 +8257,,,11507,618500,614800,356,,,3184,3153 +8423,,,11096,363000,363400,253,,,2578,2538 +8455,,,11024,1832500,1832500,550,,,8058,7411 +8760,,,11579,671900,671900,393,,,3328,3232 +8911,,,11020,1083700,1083500,490,,,4872,4811 +9017,,,11804,539200,539200,317,,,2875,2872 +9075,,,11514,470600,470000,312,,,2798,2776 +9511,,,11732,608000,608000,317,,,3223,3219 +9971,,,11568,2270200,2260000,486,,,9331,9147 +10353,,,11569,780200,780200,506,,,3288,3248 +11044,,,11211,1452300,988400,,4.05215405961764,,2911,2951 +11047,,,11355,805200,,480,2.36858974358974,,2374,2249 +11049,,,11354,841700,,504,2.50963806158137,,2382,2231 +11050,,,11374,772100,,436,2.20026245016479,,2416,2246 +11060,,,11235,690800,451500,497,2.28855721393035,,2345,2200 +11068,,,11201,4130600,1054400,1111,4.3650825281495,,2998,2954 +11070,,,11229,632300,550900,408,1.64473684210526,,2211,2161 +11262,,,11222,1598200,966400,1047,3.54444444444444,,2810,2876 +11426,,,11105,812500,673800,529,2.49745158002039,,2422,2428 +11766,,,11101,967100,704800,,4.0138303205032,4.01834523569248,2473,2565 +11776,,,11375,945400,,540,2.25,2.18245782911624,2519,2263 +\. + + CREATE SCHEMA IF NOT EXISTS "observatory"; + ALTER TABLE "obs_8764a6b439a4f8714f54d4b3a157bc5e36519066" SET SCHEMA "observatory";DROP TABLE IF EXISTS "observatory"."obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb"; CREATE TABLE "obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb" ( cartodb_id NUMERIC, the_geom GEOMETRY, the_geom_webmercator GEOMETRY, geoid TEXT, total_pop NUMERIC, male_pop NUMERIC, female_pop NUMERIC, median_age NUMERIC, white_pop NUMERIC, black_pop NUMERIC, asian_pop NUMERIC, hispanic_pop NUMERIC, amerindian_pop NUMERIC, other_race_pop NUMERIC, two_or_more_races_pop NUMERIC, not_hispanic_pop NUMERIC, not_us_citizen_pop NUMERIC, workers_16_and_over NUMERIC, commuters_by_car_truck_van NUMERIC, commuters_drove_alone NUMERIC, commuters_by_carpool NUMERIC, commuters_by_public_transportation NUMERIC, commuters_by_bus NUMERIC, commuters_by_subway_or_elevated NUMERIC, walked_to_work NUMERIC, worked_at_home NUMERIC, children NUMERIC, households NUMERIC, population_3_years_over NUMERIC, in_school NUMERIC, in_grades_1_to_4 NUMERIC, in_grades_5_to_8 NUMERIC, in_grades_9_to_12 NUMERIC, in_undergrad_college NUMERIC, pop_25_years_over NUMERIC, high_school_diploma NUMERIC, less_one_year_college NUMERIC, one_year_more_college NUMERIC, associates_degree NUMERIC, bachelors_degree NUMERIC, masters_degree NUMERIC, pop_5_years_over NUMERIC, speak_only_english_at_home NUMERIC, speak_spanish_at_home NUMERIC, pop_determined_poverty_status NUMERIC, poverty NUMERIC, median_income NUMERIC, gini_index NUMERIC, income_per_capita NUMERIC, housing_units NUMERIC, vacant_housing_units NUMERIC, vacant_housing_units_for_rent NUMERIC, vacant_housing_units_for_sale NUMERIC, median_rent NUMERIC, percent_income_spent_on_rent NUMERIC, owner_occupied_housing_units NUMERIC, million_dollar_housing_units NUMERIC, mortgaged_housing_units NUMERIC, families_with_young_children NUMERIC, two_parent_families_with_young_children NUMERIC, two_parents_in_labor_force_families_with_young_children NUMERIC, two_parents_father_in_labor_force_families_with_young_children NUMERIC, two_parents_mother_in_labor_force_families_with_young_children NUMERIC, two_parents_not_in_labor_force_families_with_young_children NUMERIC, one_parent_families_with_young_children NUMERIC, father_one_parent_families_with_young_children NUMERIC, pop_16_over NUMERIC, pop_in_labor_force NUMERIC, civilian_labor_force NUMERIC, employed_pop NUMERIC, unemployed_pop NUMERIC, armed_forces NUMERIC, not_in_labor_force NUMERIC, men_45_to_64 NUMERIC, men_45_to_49 NUMERIC, men_50_to_54 NUMERIC, men_55_to_59 NUMERIC, men_60_61 NUMERIC, men_62_64 NUMERIC, black_men_45_54 NUMERIC, black_men_55_64 NUMERIC, hispanic_men_45_54 NUMERIC, hispanic_men_55_64 NUMERIC, white_men_45_54 NUMERIC, white_men_55_64 NUMERIC, asian_men_45_54 NUMERIC, asian_men_55_64 NUMERIC, men_45_64_less_than_9_grade NUMERIC, men_45_64_grade_9_12 NUMERIC, men_45_64_high_school NUMERIC, men_45_64_some_college NUMERIC, men_45_64_associates_degree NUMERIC, men_45_64_bachelors_degree NUMERIC, men_45_64_graduate_degree NUMERIC, father_in_labor_force_one_parent_families_with_young_children NUMERIC, pop_15_and_over NUMERIC, pop_never_married NUMERIC, pop_now_married NUMERIC, pop_separated NUMERIC, pop_widowed NUMERIC, pop_divorced NUMERIC, commuters_16_over NUMERIC, commute_less_10_mins NUMERIC, commute_10_14_mins NUMERIC, commute_15_19_mins NUMERIC, commute_20_24_mins NUMERIC, commute_25_29_mins NUMERIC, commute_30_34_mins NUMERIC, commute_35_44_mins NUMERIC, commute_45_59_mins NUMERIC, commute_60_more_mins NUMERIC, aggregate_travel_time_to_work NUMERIC, income_less_10000 NUMERIC, income_10000_14999 NUMERIC, income_15000_19999 NUMERIC, income_20000_24999 NUMERIC, income_25000_29999 NUMERIC, income_30000_34999 NUMERIC, income_35000_39999 NUMERIC, income_40000_44999 NUMERIC, income_45000_49999 NUMERIC, income_50000_59999 NUMERIC, income_60000_74999 NUMERIC, income_75000_99999 NUMERIC, income_100000_124999 NUMERIC, income_125000_149999 NUMERIC, income_150000_199999 NUMERIC, income_200000_or_more NUMERIC diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index d0fbf93..4da61b7 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -40,8 +40,8 @@ SELECT Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], '2009 - 2013') a ) -select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_1, - (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_2 +select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null,"boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_1, + (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null,"boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_2 from result; -- should be null-valued diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 0899926..17cb253 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -135,13 +135,13 @@ WITH result as ( -- Point-based OBS_GetMeasure with zillow SELECT abs(OBS_GetMeasure_zhvi_point - 583600) / 583600 < 0.001 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure( - ST_SetSRID(ST_Point( -73.94602417945862, 40.6768220087458), 4326), + ST_SetSRID(ST_Point(-73.94602417945862, 40.6768220087458), 4326), 'us.zillow.AllHomes_Zhvi', 'area', 'us.census.tiger.zcta5', '2014-01' ) As t(OBS_GetMeasure_zhvi_point); -- Point-based OBS_GetMeasure with zillow default to latest -SELECT abs(OBS_GetMeasure_zhvi_point_default_latest - 583600) / 583600 < 0.001 AS OBS_GetMeasure_zhvi_point_default_latest_test FROM cdb_observatory.OBS_GetMeasure( - ST_SetSRID(ST_Point( -73.94602417945862, 40.6768220087458), 4326), +SELECT abs(OBS_GetMeasure_zhvi_point_default_latest - 972900) / 972900 < 0.001 AS OBS_GetMeasure_zhvi_point_default_latest_test FROM cdb_observatory.OBS_GetMeasure( + ST_SetSRID(ST_Point(-73.94602417945862, 40.6768220087458), 4326), 'us.zillow.AllHomes_Zhvi' ) As t(OBS_GetMeasure_zhvi_point_default_latest);