cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix minor bug in _obs_getgeometryscores with FIRST, add tests

Browse Source
This commit is contained in:
John Krauss 2016-11-07 21:26:44 +00:00
parent 9fdca9161c
commit 8dad88a6b3
6 changed files with 681 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/generate_fixtures.py
View File

@ -61,7 +61,7 @@ def get_tablename_query(column_id, boundary_id, timespan):
METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
'obs_meta_timespan', ]
'obs_meta_timespan', 'obs_column_table_tile', ]
FIXTURES = [
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
@ -290,7 +290,7 @@ def main():
"column_id IN ('{}', '{}')".format(numer_id, geom_id)
for numer_id, geom_id, timespan in FIXTURES
])
elif tablename == 'obs_column_table':
elif tablename in ('obs_column_table', 'obs_column_table_tile', ):
where = 'WHERE column_id IN ({numer_ids}) ' \
'OR column_id IN ({geom_ids}) ' \
'OR table_id IN ({table_ids}) '.format(

20
src/pg/sql/42_observatory_exploration.sql
View File

@ -454,26 +454,26 @@ BEGIN
GROUP BY column_id, table_id
) SELECT
count(*)::BIGINT, a.column_id
, (CASE WHEN FIRST(notnull_pixels) > 0
THEN FIRST(notnull_pixels) / FIRST(pixels)
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN cdb_observatory.FIRST(notnull_pixels) / cdb_observatory.FIRST(pixels)
ELSE 1
END)::Numeric AS notnull_percent
, (CASE WHEN FIRST(notnull_pixels) > 0
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum
ELSE COALESCE(ST_Value(FIRST(tile), 2, ST_PointOnSurface($1)), 0)
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0)
END)::Numeric AS numgeoms
, (CASE WHEN FIRST(notnull_pixels) > 0
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean
ELSE COALESCE(ST_Value(FIRST(tile), 3, ST_PointOnSurface($1)), 0)
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 3, ST_PointOnSurface($1)), 0)
END)::Numeric AS percentfill
, ((ST_Area(ST_Transform($1, 3857)) / 1000000) / NullIf(
CASE WHEN FIRST(notnull_pixels) > 0
CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
ELSE Coalesce(ST_Value(FIRST(tile), 1, ST_PointOnSurface($1)), 0)
ELSE Coalesce(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
END, 0))::Numeric AS estnumgeoms
, (CASE WHEN FIRST(notnull_pixels) > 0
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
ELSE COALESCE(ST_Value(FIRST(tile), 1, ST_PointOnSurface($1)), 0)
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
END)::Numeric AS meanmediansize
FROM clipped_geom_countagg a, clipped_geom b
WHERE a.table_id = b.table_id

30
src/pg/test/expected/42_observatory_exploration_test.out
View File

@ -144,6 +144,36 @@ t
_obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014
t
(1 row)
_obs_geometryscores_500m_buffer
t
(1 row)
_obs_geometryscores_5km_buffer
t
(1 row)
_obs_geometryscores_50km_buffer
t
(1 row)
_obs_geometryscores_500km_buffer
t
(1 row)
_obs_geometryscores_2500km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_500m_buffer
t
(1 row)
_obs_geometryscores_numgeoms_5km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_50km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_500km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_2500km_buffer
t
(1 row)
_total_pop_in_legacy_builder_metadata
t
(1 row)

1
src/pg/test/fixtures/drop_fixtures.sql vendored
View File

@ -12,6 +12,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_numer;
DROP TABLE IF EXISTS observatory.obs_meta_denom;
DROP TABLE IF EXISTS observatory.obs_meta_geom;
DROP TABLE IF EXISTS observatory.obs_meta_timespan;
DROP TABLE IF EXISTS observatory.obs_column_table_tile;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;

771
src/pg/test/fixtures/load_fixtures.sql vendored
View File

File diff suppressed because one or more lines are too long

92
src/pg/test/sql/42_observatory_exploration_test.sql
View File

@ -346,7 +346,98 @@ FROM cdb_observatory.OBS_GetAvailableTimespans(
) WHERE valid_denom = True)
AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;
--
-- _OBS_GetGeometryScores tests
--
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']
AS _obs_geometryscores_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']
AS _obs_geometryscores_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.county',
'us.census.tiger.zcta5', 'us.census.tiger.block_group']
AS _obs_geometryscores_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY[ 'us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
= '{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }'
AS _obs_geometryscores_numgeoms_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }'
AS _obs_geometryscores_numgeoms_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 13 }'
AS _obs_geometryscores_numgeoms_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 48415, "us.census.tiger.census_tract" : 15776, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 295 }'
AS _obs_geometryscores_numgeoms_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 165489, "us.census.tiger.census_tract" : 55152, "us.census.tiger.zcta5" : 26500, "us.census.tiger.county" : 2551 }'
AS _obs_geometryscores_numgeoms_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
--
-- OBS_LegacyBuilderMetadata tests
--
SELECT 'us.census.acs.B01003001' IN (SELECT
(jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
@ -367,4 +458,3 @@ SELECT 'us.census.acs.B19013001' NOT IN (SELECT
(jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
FROM cdb_observatory.OBS_LegacyBuilderMetadata('sum')
) AS _median_income_not_in_legacy_builder_metadata_sums;