cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'overpass' into release-v-1.3.0

Browse Source
This commit is contained in:
John Krauss 2017-01-18 18:28:53 +00:00
commit a18d88a85f
26 changed files with 12788 additions and 16035 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -1,6 +1,7 @@
include ./Makefile.global include ./Makefile.global
EXT_DIR = src/pg EXT_DIR = src/pg
PYP_DIR = src/python
.PHONY: install .PHONY: install
.PHONY: run_tests .PHONY: run_tests
@ -12,14 +13,17 @@ EXT_DIR = src/pg
# It requires sudo. # It requires sudo.
install: ## Generate and install development version of the extension; requires sudo. install: ## Generate and install development version of the extension; requires sudo.
$(MAKE) -C $(EXT_DIR) install $(MAKE) -C $(EXT_DIR) install
$(MAKE) -C $(PYP_DIR) install
# Run the tests for the installed development extension. # Run the tests for the installed development extension.
test: ## Run the tests for the development version of the extension test: ## Run the tests for the development version of the extension
$(MAKE) -C $(EXT_DIR) test $(MAKE) -C $(EXT_DIR) test
$(MAKE) -C $(PYP_DIR) test
# Generate a new release into release # Generate a new release into release
release: ## Generate a new release of the extension. Only for release manager release: ## Generate a new release of the extension. Only for release manager
$(MAKE) -C $(EXT_DIR) release $(MAKE) -C $(EXT_DIR) release
$(MAKE) -C $(PYP_DIR) release
# Install the current release. # Install the current release.
# It requires sudo. # It requires sudo.
@ -27,6 +31,7 @@ release: ## Generate a new release of the extension. Only for release manager
# sudo make deploy RELEASE_VERSION=1.0.0 # sudo make deploy RELEASE_VERSION=1.0.0
deploy: ## Deploy a released extension. Only for release manager. Requires sudo. deploy: ## Deploy a released extension. Only for release manager. Requires sudo.
$(MAKE) -C $(EXT_DIR) deploy $(MAKE) -C $(EXT_DIR) deploy
$(MAKE) -C $(PYP_DIR) deploy
# Cleanup development extension script files # Cleanup development extension script files
clean-dev: ## clean up development extension script files clean-dev: ## clean up development extension script files

151
scripts/generate_fixtures.py
View File

@ -38,24 +38,15 @@ def get_tablename_query(column_id, boundary_id, timespan):
timespan, give back the current table hash from the data observatory timespan, give back the current table hash from the data observatory
""" """
return """ return """
SELECT t.tablename, geoid_ct.colname colname, t.id table_id SELECT numer_tablename, numer_geomref_colname, numer_tid,
FROM observatory.obs_table t, geom_tablename, geom_geomref_colname, geom_tid
observatory.obs_column_table geoid_ct, FROM observatory.obs_meta
observatory.obs_column_table data_ct WHERE numer_id = '{numer_id}' AND
WHERE geom_id = '{geom_id}' AND
t.id = geoid_ct.table_id AND numer_timespan = '{numer_timespan}'
t.id = data_ct.table_id AND """.format(numer_id=column_id,
geoid_ct.column_id = geom_id=boundary_id,
(SELECT source_id numer_timespan=timespan)
FROM observatory.obs_column_to_column
WHERE target_id = '{boundary_id}'
AND reltype = 'geom_ref'
) AND
data_ct.column_id = '{column_id}' AND
timespan = '{timespan}'
""".format(column_id=column_id,
boundary_id=boundary_id,
timespan=timespan)
METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag', METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
@ -76,7 +67,7 @@ FIXTURES = [
('us.census.acs.B05001006_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B05001006_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08006001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006002_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08006002_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006008_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08301010_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006009_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08006009_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006011_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08006011_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08006015_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B08006015_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
@ -130,7 +121,6 @@ FIXTURES = [
('us.census.acs.B15003022', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B15003022', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B15003023', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B15003023', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19013001', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19013001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19083001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19301001', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19301001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B25001001', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B25001001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B25002003', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B25002003', 'us.census.tiger.block_group', '2010 - 2014'),
@ -143,15 +133,6 @@ FIXTURES = [
('us.census.acs.B25081002', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B25081002', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134001', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B08134001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134002', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B08134002', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134003', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134004', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134005', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134006', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134007', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134008', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134009', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08134010', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B08135001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001002', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001002', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001003', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001003', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001004', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001004', 'us.census.tiger.block_group', '2010 - 2014'),
@ -168,27 +149,72 @@ FIXTURES = [
('us.census.acs.B19001015', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001015', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001016', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001016', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B19001017', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B19001017', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2015'),
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
('us.census.tiger.block_group', 'us.census.tiger.block_group', '2015'),
('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2015'),
('us.census.tiger.county', 'us.census.tiger.county', '2015'),
('us.census.acs.B01001002', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B01001002', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01001002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B01003001', 'us.census.tiger.block_group', '2010 - 2014'), ('us.census.acs.B01001026', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.spielman_singleton_segments.X2', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01002001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002003', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002004', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002006', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002012', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002005', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002008', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002009', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B03002002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B11001001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003017', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003019', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003020', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003021', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003022', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B15003023', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19013001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19083001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19301001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25001001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25002003', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25004002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25004004', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25058001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25071001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25075001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25075025', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B25081002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08134001', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08134002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08134008', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08134008', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B08134010', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001002', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001003', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001004', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001005', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001006', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001007', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001008', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001009', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001010', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001011', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001012', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001013', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001014', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001015', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001016', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.acs.B19001017', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.spielman_singleton_segments.X31', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.census.spielman_singleton_segments.X55', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.spielman_singleton_segments.X55', 'us.census.tiger.census_tract', '2010 - 2014'),
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'), ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'),
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-06'), ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-06'),
('whosonfirst.wof_country_geom', 'whosonfirst.wof_country_geom', '2016'), ('whosonfirst.wof_country_name', 'whosonfirst.wof_country_geom', '2016'),
('us.census.tiger.zcta5_clipped', 'us.census.tiger.zcta5_clipped', '2014'), ('us.census.acs.B01003001', 'us.census.tiger.zcta5_clipped', '2010 - 2014'),
('us.census.tiger.block_group_clipped', 'us.census.tiger.block_group_clipped', '2014'), ('us.census.acs.B01003001', 'us.census.tiger.block_group_clipped', '2010 - 2014'),
('us.census.tiger.census_tract_clipped', 'us.census.tiger.census_tract_clipped', '2014'), ('us.census.acs.B01003001', 'us.census.tiger.census_tract_clipped', '2010 - 2014'),
('us.census.tiger.pointlm', 'us.census.tiger.pointlm', '2016'), ('us.census.tiger.fullname', 'us.census.tiger.pointlm_geom', '2016'),
('us.census.tiger.prisecroads', 'us.census.tiger.prisecroads', '2016'), ('us.census.tiger.fullname', 'us.census.tiger.prisecroads_geom', '2016'),
('us.census.tiger.name', 'us.census.tiger.county', '2015'),
('us.census.tiger.name', 'us.census.tiger.block_group', '2015'),
] ]
OUTFILE_PATH = os.path.join(os.path.dirname(__file__), '..', OUTFILE_PATH = os.path.join(os.path.dirname(__file__), '..',
@ -238,15 +264,19 @@ def main():
tablename_query = get_tablename_query(column_id, boundary_id, timespan) tablename_query = get_tablename_query(column_id, boundary_id, timespan)
resp = query(tablename_query).fetchone() resp = query(tablename_query).fetchone()
if resp: if resp:
tablename, colname, table_id = resp numer_tablename, numer_colname, numer_table_id = resp[0:3]
geom_tablename, geom_colname, geom_table_id = resp[3:6]
else: else:
print("Could not find table for {}, {}, {}".format( raise Exception("Could not find table for {}, {}, {}".format(
column_id, boundary_id, timespan)) column_id, boundary_id, timespan))
continue numer = (numer_tablename, numer_colname, numer_table_id, )
table_colname = (tablename, colname, boundary_id, table_id, ) geom = (geom_tablename, geom_colname, geom_table_id, )
if table_colname not in unique_tables: if numer not in unique_tables:
print(table_colname) print(numer)
unique_tables.add(table_colname) unique_tables.add(numer)
if geom not in unique_tables:
print(geom)
unique_tables.add(geom)
print unique_tables print unique_tables
@ -297,12 +327,12 @@ def main():
]) ])
elif tablename in ('obs_column_table', 'obs_column_table_tile', elif tablename in ('obs_column_table', 'obs_column_table_tile',
'obs_column_table_tile_simple'): 'obs_column_table_tile_simple'):
where = 'WHERE column_id IN ({numer_ids}) ' \ where = '''WHERE table_id IN ({table_ids}) AND
'OR column_id IN ({geom_ids}) ' \ (column_id IN ({numer_ids}) OR column_id IN ({geom_ids}))
'OR table_id IN ({table_ids}) '.format( '''.format(
numer_ids=','.join(["'{}'".format(x) for x, _, _ in FIXTURES]), numer_ids=','.join(["'{}'".format(x) for x, _, _ in FIXTURES]),
geom_ids=','.join(["'{}'".format(x) for _, x, _ in FIXTURES]), geom_ids=','.join(["'{}'".format(x) for _, x, _ in FIXTURES]),
table_ids=','.join(["'{}'".format(x) for _, _, _, x in unique_tables]) table_ids=','.join(["'{}'".format(x) for _, _, x in unique_tables])
) )
elif tablename == 'obs_column_to_column': elif tablename == 'obs_column_to_column':
where = "WHERE " + " OR ".join([ where = "WHERE " + " OR ".join([
@ -314,21 +344,20 @@ def main():
where = 'WHERE timespan IN ({timespans}) ' \ where = 'WHERE timespan IN ({timespans}) ' \
'OR id IN ({table_ids}) '.format( 'OR id IN ({table_ids}) '.format(
timespans=','.join(["'{}'".format(x) for _, _, x in FIXTURES]), timespans=','.join(["'{}'".format(x) for _, _, x in FIXTURES]),
table_ids=','.join(["'{}'".format(x) for _, _, _, x in unique_tables]) table_ids=','.join(["'{}'".format(x) for _, _, x in unique_tables])
) )
else: else:
where = '' where = ''
dump('*', tablename, where) dump('*', tablename, where)
for tablename, colname, boundary_id, table_id in unique_tables: for tablename, colname, table_id in unique_tables:
if 'zcta5' in boundary_id: if 'zcta5' in table_id or 'zillow_zip' in table_id:
where = '\'11%\'' where = '\'11%\''
compare = 'LIKE' compare = 'LIKE'
elif boundary_id in ('us.census.tiger.prisecroads', elif 'pri_sec_roads' in table_id or 'point_landmark' in table_id:
'us.census.tiger.pointlm'):
dump('*', tablename, 'WHERE geom && ST_MakeEnvelope(-74,40.69,-73.9,40.72, 4326)') dump('*', tablename, 'WHERE geom && ST_MakeEnvelope(-74,40.69,-73.9,40.72, 4326)')
continue continue
elif 'whosonfirst' in boundary_id: elif 'whosonfirst' in table_id:
where = '(\'85632785\',\'85633051\',\'85633111\',\'85633147\',\'85633253\',\'85633267\')' where = '(\'85632785\',\'85633051\',\'85633111\',\'85633147\',\'85633253\',\'85633267\')'
compare = 'IN' compare = 'IN'
else: else:

2
src/pg/observatory.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB Observatory backend extension' comment = 'CartoDB Observatory backend extension'
default_version = '1.1.6' default_version = '1.1.6'
requires = 'postgis, postgres_fdw' requires = 'postgis, plpythonu'
superuser = true superuser = true
schema = cdb_observatory schema = cdb_observatory

1
src/pg/sql/40_observatory_utility.sql
View File

@ -219,4 +219,3 @@ CREATE AGGREGATE cdb_observatory.FIRST (
basetype = anyelement, basetype = anyelement,
stype = anyelement stype = anyelement
); );

1325
src/pg/sql/41_observatory_augmentation.sql
View File

File diff suppressed because it is too large Load Diff

2
src/pg/sql/42_observatory_exploration.sql
View File

@ -485,7 +485,7 @@ BEGIN
, null::numeric meanmediansize , null::numeric meanmediansize
FROM clipped_geom_reagg FROM clipped_geom_reagg
) SELECT ) SELECT
((100.0 / (1+abs(log(1 + $3) - log(1 + numgeoms::Numeric)))) * percentfill)::Numeric ((100.0 / (1+abs(log(0.0001 + $3) - log(0.0001 + numgeoms::Numeric)))) * percentfill)::Numeric
AS score, * AS score, *
FROM final FROM final
$string$ USING bounds, filter_geom_ids, desired_num_geoms; $string$ USING bounds, filter_geom_ids, desired_num_geoms;

258
src/pg/sql/44_observatory_geometries.sql
View File

@ -40,44 +40,11 @@ BEGIN
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom); RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom);
END IF; END IF;
-- choose appropriate table based on time_span
IF time_span IS NULL
THEN
SELECT x.target_tables INTO target_table
FROM cdb_observatory._OBS_SearchTables(boundary_id,
time_span) As x(target_tables,
timespans)
ORDER BY x.timespans DESC
LIMIT 1;
ELSE
-- TODO: modify for only one table returned instead of arbitrarily choosing
-- one with LIMIT 1 (could be conflict between clipped vs non-clipped
-- boundaries in the metadata tables)
SELECT x.target_tables INTO target_table
FROM cdb_observatory._OBS_SearchTables(boundary_id,
time_span) As x(target_tables,
timespans)
WHERE x.timespans = time_span
LIMIT 1;
END IF;
-- if no tables are found, raise notice and return null
IF target_table IS NULL
THEN
--RAISE NOTICE 'No boundaries found for ''%'' in ''%''', ST_AsText(geom), boundary_id;
RETURN NULL::geometry;
END IF;
--RAISE NOTICE 'target_table: %', target_table;
-- return the first boundary in intersections -- return the first boundary in intersections
EXECUTE format( EXECUTE $query$
'SELECT the_geom SELECT * FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2, $3) LIMIT 1
FROM observatory.%I $query$ INTO boundary
WHERE ST_Intersects($1, the_geom) USING geom, boundary_id, time_span;
LIMIT 1', target_table)
INTO boundary
USING geom;
RETURN boundary; RETURN boundary;
@ -111,67 +78,17 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryId(
RETURNS text RETURNS text
AS $$ AS $$
DECLARE DECLARE
output_id text; result TEXT;
target_table text;
geoid_colname text;
BEGIN BEGIN
-- If not point, raise error EXECUTE $query$
IF ST_GeometryType(geom) != 'ST_Point' SELECT geom_refs FROM cdb_observatory._OBS_GetBoundariesByGeometry(
THEN $1, $2, $3) LIMIT 1
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom); $query$
END IF; INTO result
USING geom, boundary_id, time_span;
-- choose appropriate table based on time_span
IF time_span IS NULL
THEN
SELECT x.target_tables INTO target_table
FROM cdb_observatory._OBS_SearchTables(boundary_id,
time_span) As x(target_tables,
timespans)
ORDER BY x.timespans DESC
LIMIT 1;
ELSE
SELECT x.target_tables INTO target_table
FROM cdb_observatory._OBS_SearchTables(boundary_id,
time_span) As x(target_tables,
timespans)
WHERE x.timespans = time_span
LIMIT 1;
END IF;
-- if no tables are found, raise notice and return null
IF target_table IS NULL
THEN
--RAISE NOTICE 'Warning: No boundaries found for ''%''', boundary_id;
RETURN NULL::text;
END IF;
EXECUTE
'SELECT ct.colname
FROM observatory.obs_column_to_column c2c,
observatory.obs_column_table ct,
observatory.obs_table t
WHERE c2c.reltype = ''geom_ref''
AND ct.column_id = c2c.source_id
AND ct.table_id = t.id
AND t.tablename = $1'
INTO geoid_colname
USING target_table;
--RAISE NOTICE 'target_table: %, geoid_colname: %', target_table, geoid_colname;
-- return geometry id column value
EXECUTE format(
'SELECT %I::text
FROM observatory.%I
WHERE ST_Intersects($1, the_geom)
LIMIT 1', geoid_colname, target_table)
INTO output_id
USING geom;
RETURN output_id;
RETURN result;
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
@ -203,35 +120,21 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryById(
RETURNS geometry(geometry, 4326) RETURNS geometry(geometry, 4326)
AS $$ AS $$
DECLARE DECLARE
boundary geometry(geometry, 4326); result GEOMETRY;
target_table text;
geoid_colname text;
geom_colname text;
BEGIN BEGIN
SELECT * INTO geoid_colname, target_table, geom_colname EXECUTE $query$
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id); SELECT (data->0->>'value')::Geometry
FROM cdb_observatory.OBS_GetData(
--RAISE NOTICE '%', target_table; ARRAY[$1],
cdb_observatory.OBS_GetMeta(
IF target_table IS NULL ST_MakeEnvelope(-180, -90, 180, 90, 4326),
THEN ('[{"geom_id": "' || $2 || '"}]')::JSON))
--RAISE NOTICE 'No geometries found'; $query$
RETURN NULL::geometry; INTO result
END IF; USING geometry_id, boundary_id;
-- retrieve boundary
EXECUTE
format(
'SELECT %I
FROM observatory.%I
WHERE %I = $1
LIMIT 1', geom_colname, target_table, geoid_colname)
INTO boundary
USING geometry_id;
RETURN boundary;
RETURN result;
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
@ -245,13 +148,12 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetBoundariesByGeometry(
boundary_id text, boundary_id text,
time_span text DEFAULT NULL, time_span text DEFAULT NULL,
overlap_type text DEFAULT NULL) overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) RETURNS TABLE (
AS $$ the_geom geometry,
geom_refs text
) AS $$
DECLARE DECLARE
boundary geometry(Geometry, 4326); meta JSON;
geom_colname text;
geoid_colname text;
target_table text;
BEGIN BEGIN
overlap_type := COALESCE(overlap_type, 'intersects'); overlap_type := COALESCE(overlap_type, 'intersects');
-- check inputs -- check inputs
@ -259,34 +161,27 @@ BEGIN
THEN THEN
-- recognized overlap type (map to ST_Contains, ST_Intersects, and ST_Within) -- recognized overlap type (map to ST_Contains, ST_Intersects, and ST_Within)
RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type; RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type;
ELSIF ST_GeometryType(geom) NOT IN ('ST_Polygon', 'ST_MultiPolygon')
THEN
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
END IF; END IF;
-- TODO: add timespan in search EXECUTE $query$
-- TODO: add overlap info in search SELECT cdb_observatory.OBS_GetMeta($1, JSON_Build_Array(JSON_Build_Object(
SELECT * INTO geoid_colname, target_table, geom_colname 'geom_id', $2, 'geom_timespan', $3)))
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id); $query$
INTO meta
USING geom, boundary_id, time_span;
-- if no tables are found, raise notice and return null IF meta->0->>'geom_id' IS NULL THEN
IF target_table IS NULL RETURN QUERY EXECUTE 'SELECT NULL::Geometry, NULL::Text LIMIT 0';
THEN
--RAISE NOTICE 'No boundaries found for bounding box ''%'' in ''%''', ST_AsText(geom), boundary_id;
RETURN QUERY SELECT NULL::geometry, NULL::text;
RETURN; RETURN;
END IF; END IF;
--RAISE NOTICE 'target_table: %', target_table;
-- return first boundary in intersections -- return first boundary in intersections
RETURN QUERY RETURN QUERY EXECUTE $query$
EXECUTE format( SELECT (data->0->>'value')::Geometry the_geom, data->0->>'geomref' geom_refs
'SELECT %I, %I::text FROM cdb_observatory.OBS_GetData(
FROM observatory.%I ARRAY[($1, 1)::geomval], $2, False
WHERE ST_%s($1, %I) )
', geom_colname, geoid_colname, target_table, overlap_type, geom_colname) $query$ USING geom, meta;
USING geom;
RETURN; RETURN;
END; END;
@ -414,27 +309,11 @@ BEGIN
RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom); RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
END IF; END IF;
SELECT * INTO geoid_colname, target_table, geom_colname
FROM cdb_observatory._OBS_GetGeometryMetadata(boundary_id);
-- if no tables are found, raise notice and return null
IF target_table IS NULL
THEN
--RAISE NOTICE 'No boundaries found for bounding box ''%'' in ''%''', ST_AsText(geom), boundary_id;
RETURN QUERY SELECT NULL::geometry, NULL::text;
RETURN;
END IF;
--RAISE NOTICE 'target_table: %', target_table;
-- return first boundary in intersections -- return first boundary in intersections
RETURN QUERY RETURN QUERY EXECUTE $query$
EXECUTE format( SELECT ST_PointOnSurface(the_geom), geom_refs
'SELECT ST_PointOnSurface(%I) As %s, %I::text FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2)
FROM observatory.%I $query$ USING geom, boundary_id;
WHERE ST_%s($1, the_geom)
', geom_colname, geom_colname, geoid_colname, target_table, overlap_type)
USING geom;
RETURN; RETURN;
END; END;
@ -534,46 +413,3 @@ BEGIN
RETURN; RETURN;
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
-- _OBS_GetGeometryMetadata()
-- TODO: add timespan in search
-- TODO: add choice of clipped versus not clipped
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryMetadata(boundary_id text)
RETURNS table(geoid_colname text, target_table text, geom_colname text)
AS $$
BEGIN
RETURN QUERY
EXECUTE $string$
SELECT geoid_ct.colname::text As geoid_colname,
tablename::text,
geom_ct.colname::text As geom_colname
FROM observatory.obs_column_table As geoid_ct,
observatory.obs_table As geom_t,
observatory.obs_column_table As geom_ct,
observatory.obs_column As geom_c
WHERE geoid_ct.column_id
IN (
SELECT source_id
FROM observatory.obs_column_to_column
WHERE reltype = 'geom_ref'
AND target_id = $1
)
AND geoid_ct.table_id = geom_t.id AND
geom_t.id = geom_ct.table_id AND
geom_ct.column_id = geom_c.id AND
geom_c.type ILIKE 'geometry%' AND
geom_c.id = $1
ORDER BY timespan DESC
LIMIT 1
$string$
USING boundary_id;
RETURN;
-- AND geom_t.timespan = '%s' <-- put in requested year
-- TODO: filter by clipped vs. not so appropriate tablename are unique
-- so the limit 1 can be removed
RETURN;
END;
$$ LANGUAGE plpgsql;

82
src/pg/sql/50_table_level_framework.sql
View File

@ -1,82 +0,0 @@
CREATE TYPE cdb_observatory.ds_fdw_metadata as (schemaname text, tabname text, servername text);
CREATE TYPE cdb_observatory.ds_return_metadata as (colnames text[], coltypes text[]);
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_observatory.ds_fdw_metadata
AS $$
DECLARE
fdw_server text;
fdw_import_schema text;
connection_str json;
import_foreign_schema_q text;
epoch_timestamp text;
BEGIN
SELECT extract(epoch from now() at time zone 'utc')::int INTO epoch_timestamp;
fdw_server := 'fdw_server_' || username || '_' || epoch_timestamp;
fdw_import_schema:= fdw_server;
-- Import foreign table
EXECUTE FORMAT ('SELECT cdb_observatory._OBS_ConnectRemoteTable(%L, %L, %L, %L, %L, %L, %L)', fdw_server, fdw_import_schema, dbname, host_addr, user_db_role, table_name, input_schema);
RETURN (fdw_import_schema::text, table_name::text, fdw_server::text);
EXCEPTION
WHEN others THEN
-- Disconnect user imported table. Delete schema and FDW server.
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".' || table_name;
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".cdb_tablemetadata';
EXECUTE 'DROP SCHEMA IF EXISTS "' || fdw_import_schema || '"';
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || fdw_server || '"';
EXECUTE 'DROP SERVER IF EXISTS "' || fdw_server || '"';
RETURN (null, null, null);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_observatory.ds_return_metadata
AS $$
DECLARE
colnames text[];
coltypes text[];
BEGIN
EXECUTE FORMAT('SELECT r.colnames::text[], r.coltypes::text[] FROM cdb_observatory._%sResultMetadata(%L::json) r', function_name, params::text)
INTO colnames, coltypes;
RETURN (colnames::text[], coltypes::text[]);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record
AS $$
DECLARE
data_query text;
rec RECORD;
BEGIN
EXECUTE FORMAT('SELECT cdb_observatory._%sQuery(%L, %L, %L::json)', function_name, table_schema, table_name, params::text)
INTO data_query;
FOR rec IN EXECUTE data_query
LOOP
RETURN NEXT rec;
END LOOP;
RETURN;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean
AS $$
BEGIN
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".' || table_name;
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".cdb_tablemetadata';
EXECUTE 'DROP SCHEMA IF EXISTS "' || table_schema || '"';
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || servername || '"';
EXECUTE 'DROP SERVER IF EXISTS "' || servername || '"';
RETURN true;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;

79
src/pg/sql/51_table_level_functions.sql
View File

@ -1,79 +0,0 @@
--
--
-- OBS_GetMeasure
--
--
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureResultMetadata(params json)
RETURNS cdb_observatory.ds_return_metadata
AS $$
DECLARE
colnames text[]; -- Array to store the name of the measures to be returned
coltypes text[]; -- Array to store the type of the measures to be returned
requested_measures text[];
measure_id text;
BEGIN
-- By definition, all the measure results for the OBS_GetMeasure API are numeric values
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO requested_measures;
FOREACH measure_id IN ARRAY requested_measures
LOOP
SELECT array_append(colnames, measure_id) INTO colnames;
SELECT array_append(coltypes, 'numeric'::text) INTO coltypes;
END LOOP;
RETURN (colnames::text[], coltypes::text[]);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureQuery(table_schema text, table_name text, params json)
RETURNS text
AS $$
DECLARE
data_query text;
measure_ids_arr text[];
measure_id text;
measures_list text;
measures_query text;
normalize text;
boundary_id text;
time_span text;
geom_table_name text;
data_table_name text;
BEGIN
measures_query := '';
-- SELECT table_name from obs_meta WHERE boundary_id = {bound} AND [...] INTO geom_table_name
geom_table_name := 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308';
-- SELECT table_name from obs_meta WHERE time_span = {time} AND [...] INTO data_table_name
data_table_name := 'observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd';
-- Get measure_ids array from JSON
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO measure_ids_arr;
-- Get a comma-separated list of measures ("total_pop, over_16_pop") to be used in SELECTs
SELECT array_to_string(measure_ids_arr, ',') INTO measures_list;
FOREACH measure_id IN ARRAY measure_ids_arr
LOOP
-- Build query to compute each value and normalize
-- Assumes the default normalization method, the normalize parameter given in the JSON
-- should be checked in order to build the final query
SELECT measures_query || ' sum(' || measure_id || '/fraction)::numeric as ' || measure_id || ', ' INTO measures_query;
END LOOP;
-- Data query should select the measures and the cartodb_id of the user table, in that order.
data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
|| '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
|| geom_table_name || ' as a, '
|| table_schema || '.' || table_name || ' AS b '
|| 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
|| measures_list
|| ' FROM ' || data_table_name || ' ) '
|| 'SELECT '
|| measures_query
|| ' cartodb_id::int FROM _areas, values '
|| 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
RETURN data_query;
END;
$$ LANGUAGE plpgsql;

107
src/pg/sql/60_observatory_providers.sql Normal file
View File

@ -0,0 +1,107 @@
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetOverpass(
query text
) RETURNS TABLE (
geom TEXT,
"type" TEXT,
id TEXT,
properties TEXT
) AS $$
from observatory.osm import get_overpass
import plpy
import json
result = get_overpass(query)
return [{
'geom': json.dumps(feature['geometry']),
'type': feature['type'],
'id': str(feature['id']),
'properties': json.dumps(feature['properties'])
} for feature in result]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetOverpass(
query text
) RETURNS TABLE (
geom Geometry(Geometry, 4326),
"type" TEXT,
id TEXT,
properties JSON
) AS $$
BEGIN
RETURN QUERY
EXECUTE $string$
SELECT ST_GeomFromGeojson(geom) geom,
"type",
id,
properties::JSON
FROM cdb_observatory._OBS_GetOverPass($1)
$string$ USING query
RETURN;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetOverpassPOI(
within_geom Geometry(Geometry, 4326),
filters TEXT[] DEFAULT ARRAY[ 'pub', 'bar', 'restaurant', 'fast_food',
'cafe', 'food_court', 'ice_cream', 'college', 'kindergarten', 'library',
'school', 'music_school', 'driving_school', 'language_school', 'university',
'bicycle_rental', 'boat_sharing', 'car_rental', 'car_sharing', 'car_wash',
'ferry_terminal', 'fuel', 'bank', 'bureau_de_change', 'clinic', 'dentist',
'doctors', 'hospital', 'nursing_home', 'pharmacy', 'social_facility',
'veterinary', 'blood_donation', 'arts_centre', 'brothel', 'casino',
'community_centre', 'cinema', 'gambling', 'nightclub', 'planetarium',
'social_centre', 'stripclub', 'swingerclub', 'studio', 'theatre',
'animal_boarding', 'animal_shelter', 'courthouse', 'coworking_space',
'crematorium', 'dive_centre', 'dojo', 'embassy', 'fire_station',
'internet_cafe', 'marketplace', 'place_of_worship', 'police', 'post_office',
'prison', 'townhall', 'waste_transfer_station'
],
name TEXT DEFAULT NULL
) RETURNS TABLE (
geom Geometry(Geometry, 4326),
"type" TEXT,
id TEXT,
properties JSON
) AS $$
DECLARE
osm_bbox TEXT;
query TEXT;
BEGIN
osm_bbox := replace(replace(replace(regexp_replace(box2d(within_geom)::TEXT,
E'(\\-?\\d+\\.?\\d+) (\\-?\\d+\\.?\\d+)',
E'\\2 \\1', 'g'),
' ', ','),
'BOX(', ''),
')', '');
EXECUTE $string$
WITH filters AS (SELECT UNNEST($1) as filter)
SELECT ' ( '
'node ["name"]["amenity"~"' || String_Agg(filter, '|') || '"] (' || $2 || '); ' ||
'way ["name"]["amenity"~"' || String_Agg(filter, '|') || '"] (' || $2 || '); ' ||
'node ["name"][shop] (' || $2 || '); ' ||
'way ["name"][shop] (' || $2 || '); ' ||
'relation ["name"][shop] (' || $2 || ')) '
FROM filters
$string$
INTO query
USING filters, osm_bbox, name;
RAISE NOTICE '%', query;
RETURN QUERY
EXECUTE $string$
WITH results AS (SELECT ST_SetSRID(ST_GeomFromGeojson(geom), 4326) geom,
"type",
id,
properties::JSON
FROM cdb_observatory._OBS_GetOverPass($1))
SELECT * FROM results WHERE ST_Within(geom, $2)
$string$ USING query, within_geom
RETURN;
END;
$$ LANGUAGE plpgsql;

1
src/pg/test/expected/01_install_test.out
View File

@ -1,5 +1,6 @@
-- Install dependencies -- Install dependencies
CREATE EXTENSION postgis; CREATE EXTENSION postgis;
CREATE EXTENSION postgres_fdw; CREATE EXTENSION postgres_fdw;
CREATE EXTENSION plpythonu;
-- Install the extension -- Install the extension
CREATE EXTENSION observatory VERSION 'dev'; CREATE EXTENSION observatory VERSION 'dev';

153
src/pg/test/expected/41_observatory_augmentation_test.out
View File

@ -3,24 +3,12 @@
obs_getdemographicsnapshot_test_no_returns obs_getdemographicsnapshot_test_no_returns
t t
(1 row) (1 row)
obs_get_median_income_at_test_point
t
(1 row)
obs_get_median_income_at_null_island
t
(1 row)
test_point_segmentation test_point_segmentation
t t
(1 row) (1 row)
null_island_segmentation null_island_segmentation
t t
(1 row) (1 row)
getcategories_at_test_point_1
t
(1 row)
getcategories_at_null_island
t
(1 row)
obs_getmeasure_zhvi_point_test obs_getmeasure_zhvi_point_test
t t
(1 row) (1 row)
@ -117,3 +105,144 @@ t
obs_getmeasurebyid_null_id obs_getmeasurebyid_null_id
t t
(1 row) (1 row)
obs_getmeta_null_null_is_null
t
(1 row)
obs_getmeta_null_empty_is_null
t
(1 row)
obs_getmeta_nullisland_null_is_null
t
(1 row)
obs_getmeta_nullisland_empty_is_null
t
(1 row)
obs_getmeta_nullisland_us_measure_is_null
t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization|id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
(1 row)
id|numer_id|timespan_rank|score_rank|numer_aggregate|numer_colname|numer_type|numer_name|denom_id|denom_aggregate|denom_colname|denom_type|denom_name|geom_id|normalization
t|t|t|t|t|t|t|t|t|t|t|t|t|t|t
(1 row)
obs_getmeta_conflicting_metadata
t
(1 row)
obs_getdata_geomval_empty_null
t
(1 row)
obs_getdata_text_empty_null
t
(1 row)
obs_getdata_geomval_empty_one_measure
t
(1 row)
id|data_point_measure_null|nullcol
t|t|t
(1 row)
id|data_polygon_measure_null|nullcol
t|t|t
(1 row)
id|data_point_measure_area|nullcol
t|t|t
(1 row)
id|data_polygon_measure_area|nullcol
t|t|t
(1 row)
id|data_point_measure_predenominated|nullcol
t|t|t
(1 row)
id|data_polygon_measure_predenominated|nullcol
t|t|t
(1 row)
id|data_point_measure_impossible_denominated|nullcol
t|t|t
(1 row)
id|data_polygon_measure_impossible_denominated|nullcol
t|t|t
(1 row)
id|data_point_measure_denominated|nullcol
t|t|t
(1 row)
id|data_polygon_measure_denominated|nullcol
t|t|t
(1 row)
id|data_polygon_measure_one_null|data_polygon_measure_two_null
t|t|t
(1 row)
id|data_polygon_measure_one_predenom|data_polygon_measure_two_predenom
t|t|t
(1 row)
id|data_polygon_measure_one_area|data_polygon_measure_two_area
t|t|t
(1 row)
id|data_polygon_measure_tract|data_polygon_measure_bg
t|t|t
(1 row)
id|data_point_categorical|nullcol
t|t|t
(1 row)
id|data_poly_categorical|nullcol
t|t|t
(1 row)
id|data_poly_categorical|valcol
t|t|t
(1 row)
id|correct_num_geoms
t|t
(1 row)
id|correct_num_geoms|correct_pop
t|t|t
(1 row)
id|correct_num_geoms|correct_pop|correct_bg_names
t|t|t|t
(1 row)
id|correct_num_points
t|t
(1 row)
id|correct_num_points|pointgeom_names
t|t|t
(1 row)
id|obs_getdata_by_id_one_measure_null
t|t
(1 row)
id|obs_getdata_by_id_one_measure_predenom
t|t
(1 row)
id|obs_getdata_by_id_one_measure_null|obs_getdata_by_id_two_measure_null
t|t|t
(1 row)
id|obs_getdata_by_id_categorical
t|t
(1 row)
id|obs_getdata_by_id_geometry
t|t
(1 row)
obs_getdata_api_geomvals_no_args
t
(1 row)
obs_getdata_api_geomvals_args_numer_return
t
(1 row)
obs_getdata_api_geomvals_args_string_return
t
(1 row)
obs_getdata_api_geomrefs_args_numer_return
t
(1 row)
obs_getdata_api_geomrefs_args_string_return
t
(1 row)

6
src/pg/test/expected/44_observatory_geometries_test.out
View File

@ -93,9 +93,3 @@ t
obs_getpointsbypointandradius_around_null_island obs_getpointsbypointandradius_around_null_island
t t
(1 row) (1 row)
geoid_name_matches|table_name_matches|geom_name_matches
t|t|t
(1 row)
geoid_name_matches|table_name_matches|geom_name_matches
t|t|t
(1 row)

28
src/pg/test/fixtures/drop_fixtures.sql vendored
View File

@ -14,20 +14,20 @@ DROP TABLE IF EXISTS observatory.obs_meta_geom;
DROP TABLE IF EXISTS observatory.obs_meta_timespan; DROP TABLE IF EXISTS observatory.obs_meta_timespan;
DROP TABLE IF EXISTS observatory.obs_column_table_tile; DROP TABLE IF EXISTS observatory.obs_column_table_tile;
DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple; DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_e32f8e59c7c8861ee5ee4029b3ace2af9a5c9caf;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
DROP TABLE IF EXISTS observatory.obs_0310c639744a2014bb1af82709228f05b59e7d3d;
DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
DROP TABLE IF EXISTS observatory.obs_08025e1287e3af2b5de571d06562ba8d3bdb48e9;
DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a;
DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c; DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c;
DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d; DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d;
DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;
DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd;
DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c;
DROP TABLE IF EXISTS observatory.obs_87a814e485deabe3b12545a537f693d16ca702c2;
DROP TABLE IF EXISTS observatory.obs_c4411eba732408d47d73281772dbf03d60645dec;
DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3; DROP TABLE IF EXISTS observatory.obs_1746e37b7cd28cb131971ea4187d42d71f09c5f3;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_c4411eba732408d47d73281772dbf03d60645dec;
DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd;
DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
DROP TABLE IF EXISTS observatory.obs_a01cd5d8ccaa6531cef715071e9307e6b1987ec3;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
DROP TABLE IF EXISTS observatory.obs_0310c639744a2014bb1af82709228f05b59e7d3d;
DROP TABLE IF EXISTS observatory.obs_87a814e485deabe3b12545a537f693d16ca702c2;
DROP TABLE IF EXISTS observatory.obs_e32f8e59c7c8861ee5ee4029b3ace2af9a5c9caf;
DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c;
DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a;
DROP TABLE IF EXISTS observatory.obs_8e30e6b3792430b410ba5b9e49cdc6a0d404d48f;
DROP TABLE IF EXISTS observatory.obs_08025e1287e3af2b5de571d06562ba8d3bdb48e9;
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;

25887
src/pg/test/fixtures/load_fixtures.sql vendored
View File

File diff suppressed because one or more lines are too long

1
src/pg/test/sql/01_install_test.sql
View File

@ -1,6 +1,7 @@
-- Install dependencies -- Install dependencies
CREATE EXTENSION postgis; CREATE EXTENSION postgis;
CREATE EXTENSION postgres_fdw; CREATE EXTENSION postgres_fdw;
CREATE EXTENSION plpythonu;
-- Install the extension -- Install the extension
CREATE EXTENSION observatory VERSION 'dev'; CREATE EXTENSION observatory VERSION 'dev';

4
src/pg/test/sql/40_observatory_utility_test.sql
View File

@ -11,8 +11,8 @@ SELECT
cdb_observatory._OBS_GeomTable( cdb_observatory._OBS_GeomTable(
ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326), ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326),
'us.census.tiger.census_tract', 'us.census.tiger.census_tract',
'2014' '2015'
) = 'obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7' As _obs_geomtable_with_returned_table; ) = 'obs_87a814e485deabe3b12545a537f693d16ca702c2' As _obs_geomtable_with_returned_table;
-- get null for unknown geometry_id -- get null for unknown geometry_id
-- should give back null -- should give back null

590
src/pg/test/sql/41_observatory_augmentation_test.sql
View File

@ -5,67 +5,21 @@ SET client_min_messages TO WARNING;
-- --
WITH result as( WITH result as(
Select count(coalesce(OBS_GetDemographicSnapshot->>'value', 'foo')) expected_columns Select count(coalesce(OBS_GetDemographicSnapshot->>'value', 'foo')) expected_columns
FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint()) FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint(), '2010 - 2014')
) select expected_columns = 52 as OBS_GetDemographicSnapshot_test_no_returns ) select expected_columns = 52 as OBS_GetDemographicSnapshot_test_no_returns
FROM result; FROM result;
WITH result as (
SELECT _OBS_Get::text as expected FROM
cdb_observatory._OBS_Get(
cdb_observatory._TestPoint(),
Array['us.census.acs.B19013001']::text[],
'2010 - 2014',
'us.census.tiger.block_group'
)
) SELECT expected = '{"value":79292,"name":"Median Household Income in the past 12 Months","tablename":"obs_1a098da56badf5f32e336002b0a81708c40d29cd","aggregate":"median","type":"Numeric","description":"Within a geographic area, the median income received by every household on a regular basis before payments for personal income taxes, social security, union dues, medicare deductions, etc. It includes income received from wages, salary, commissions, bonuses, and tips; self-employment income from own nonfarm or farm businesses, including proprietorships and partnerships; interest, dividends, net rental income, royalty income, or income from estates and trusts; Social Security or Railroad Retirement income; Supplemental Security Income (SSI); any cash public assistance or welfare payments from the state or local welfare office; retirement, survivor, or disability benefits; and any other sources of income received regularly such as Veterans'' (VA) payments, unemployment and/or worker''s compensation, child support, and alimony."}'
As OBS_Get_median_income_at_test_point
FROM result;
-- median income at null island
WITH result as (
SELECT count(_OBS_Get) as expected FROM
cdb_observatory._OBS_Get(
ST_SetSRID(ST_Point(0, 0), 4326),
Array['us.census.acs.B19013001']::text[],
'2010 - 2014',
'us.census.tiger.block_group'
)
) select expected = 0 as OBS_Get_median_income_at_null_island
from result;
SELECT cdb_observatory.OBS_GetSegmentSnapshot( SELECT cdb_observatory.OBS_GetSegmentSnapshot(
cdb_observatory._TestPoint(), cdb_observatory._TestPoint(),
'us.census.tiger.census_tract' 'us.census.tiger.census_tract'
)::text = )::JSONB =
'{"x10_segment":"Wealthy, urban without Kids","x55_segment":"Wealthy transplants displacing long-term local residents","us.census.acs.B01003001_quantile":"0.3235","us.census.acs.B01001002_quantile":"0.494716216216216","us.census.acs.B01001026_quantile":"0.183756756756757","us.census.acs.B01002001_quantile":"0.0752837837837838","us.census.acs.B03002003_quantile":"0.293162162162162","us.census.acs.B03002004_quantile":"0.455527027027027","us.census.acs.B03002006_quantile":"0.656405405405405","us.census.acs.B03002012_quantile":"0.840081081081081","us.census.acs.B05001006_quantile":"0.727135135135135","us.census.acs.B08006001_quantile":"0.688635135135135","us.census.acs.B08006002_quantile":"0.0204459459459459","us.census.acs.B08006008_quantile":"0.679324324324324","us.census.acs.B08006009_quantile":"0.996716216216216","us.census.acs.B08006011_quantile":"0.967418918918919","us.census.acs.B08006015_quantile":"0.512945945945946","us.census.acs.B08006017_quantile":"0.0504864864864865","us.census.acs.B09001001_quantile":"0.192405405405405","us.census.acs.B11001001_quantile":"0.331702702702703","us.census.acs.B14001001_quantile":"0.296283783783784","us.census.acs.B14001002_quantile":"0.045472972972973","us.census.acs.B14001005_quantile":"0.0442702702702703","us.census.acs.B14001006_quantile":"0.0829054054054054","us.census.acs.B14001007_quantile":"0.701135135135135","us.census.acs.B14001008_quantile":"0.404527027027027","us.census.acs.B15003001_quantile":"0.191824324324324","us.census.acs.B15003017_quantile":"0.864162162162162","us.census.acs.B15003022_quantile":"0.754297297297297","us.census.acs.B15003023_quantile":"0.350054054054054","us.census.acs.B16001001_quantile":"0.217635135135135","us.census.acs.B16001002_quantile":"0.85972972972973","us.census.acs.B16001003_quantile":"0.342851351351351","us.census.acs.B17001001_quantile":"0.51204054054054","us.census.acs.B17001002_quantile":"0.813540540540541","us.census.acs.B19013001_quantile":"0.0948648648648649","us.census.acs.B19083001_quantile":"0.678351351351351","us.census.acs.B19301001_quantile":"0.146108108108108","us.census.acs.B25001001_quantile":"0.149067567567568","us.census.acs.B25002003_quantile":"0","us.census.acs.B25004002_quantile":"0","us.census.acs.B25004004_quantile":"0.944554054054054","us.census.acs.B25058001_quantile":"0.398040540540541","us.census.acs.B25071001_quantile":"0.0596081081081081","us.census.acs.B25075001_quantile":"0","us.census.acs.B25075025_quantile":null}' as test_point_segmentation; '{"x10_segment": "Wealthy, urban without Kids", "x55_segment": "Wealthy transplants displacing long-term local residents", "us.census.acs.B01001002_quantile": "0.494716216216216", "us.census.acs.B01001026_quantile": "0.183756756756757", "us.census.acs.B01002001_quantile": "0.0752837837837838", "us.census.acs.B01003001_quantile": "0.3235", "us.census.acs.B03002003_quantile": "0.293162162162162", "us.census.acs.B03002004_quantile": "0.455527027027027", "us.census.acs.B03002006_quantile": "0.656405405405405", "us.census.acs.B03002012_quantile": "0.840081081081081", "us.census.acs.B05001006_quantile": "0.727135135135135", "us.census.acs.B08006001_quantile": "0.688635135135135", "us.census.acs.B08006002_quantile": "0.0204459459459459", "us.census.acs.B08006009_quantile": "0.679324324324324", "us.census.acs.B08006011_quantile": "0.996716216216216", "us.census.acs.B08006015_quantile": "0.967418918918919", "us.census.acs.B08006017_quantile": "0.512945945945946", "us.census.acs.B08301010_quantile": "0.994743243243243", "us.census.acs.B09001001_quantile": "0.0504864864864865", "us.census.acs.B11001001_quantile": "0.192405405405405", "us.census.acs.B14001001_quantile": "0.331702702702703", "us.census.acs.B14001002_quantile": "0.296283783783784", "us.census.acs.B14001005_quantile": "0.045472972972973", "us.census.acs.B14001006_quantile": "0.0442702702702703", "us.census.acs.B14001007_quantile": "0.0829054054054054", "us.census.acs.B14001008_quantile": "0.701135135135135", "us.census.acs.B15003001_quantile": "0.404527027027027", "us.census.acs.B15003017_quantile": "0.191824324324324", "us.census.acs.B15003022_quantile": "0.864162162162162", "us.census.acs.B15003023_quantile": "0.754297297297297", "us.census.acs.B16001001_quantile": "0.350054054054054", "us.census.acs.B16001002_quantile": "0.217635135135135", "us.census.acs.B16001003_quantile": "0.85972972972973", "us.census.acs.B17001001_quantile": "0.342851351351351", "us.census.acs.B17001002_quantile": "0.51204054054054", "us.census.acs.B19013001_quantile": "0.813540540540541", "us.census.acs.B19083001_quantile": "0.0948648648648649", "us.census.acs.B19301001_quantile": "0.678351351351351", "us.census.acs.B25001001_quantile": "0.146108108108108", "us.census.acs.B25002003_quantile": "0.149067567567568", "us.census.acs.B25004002_quantile": "0", "us.census.acs.B25004004_quantile": "0", "us.census.acs.B25058001_quantile": "0.944554054054054", "us.census.acs.B25071001_quantile": "0.398040540540541", "us.census.acs.B25075001_quantile": "0.0596081081081081", "us.census.acs.B25075025_quantile": "0"}'::JSONB as test_point_segmentation;
-- segmentation around null island -- segmentation around null island
SELECT cdb_observatory.OBS_GetSegmentSnapshot( SELECT cdb_observatory.OBS_GetSegmentSnapshot(
ST_SetSRID(ST_Point(0, 0), 4326), ST_SetSRID(ST_Point(0, 0), 4326),
'us.census.tiger.census_tract' 'us.census.tiger.census_tract'
)::text = '{"x10_segment":null,"x55_segment":null,"us.census.acs.B01003001_quantile":null,"us.census.acs.B01001002_quantile":null,"us.census.acs.B01001026_quantile":null,"us.census.acs.B01002001_quantile":null,"us.census.acs.B03002003_quantile":null,"us.census.acs.B03002004_quantile":null,"us.census.acs.B03002006_quantile":null,"us.census.acs.B03002012_quantile":null,"us.census.acs.B05001006_quantile":null,"us.census.acs.B08006001_quantile":null,"us.census.acs.B08006002_quantile":null,"us.census.acs.B08006008_quantile":null,"us.census.acs.B08006009_quantile":null,"us.census.acs.B08006011_quantile":null,"us.census.acs.B08006015_quantile":null,"us.census.acs.B08006017_quantile":null,"us.census.acs.B09001001_quantile":null,"us.census.acs.B11001001_quantile":null,"us.census.acs.B14001001_quantile":null,"us.census.acs.B14001002_quantile":null,"us.census.acs.B14001005_quantile":null,"us.census.acs.B14001006_quantile":null,"us.census.acs.B14001007_quantile":null,"us.census.acs.B14001008_quantile":null,"us.census.acs.B15003001_quantile":null,"us.census.acs.B15003017_quantile":null,"us.census.acs.B15003022_quantile":null,"us.census.acs.B15003023_quantile":null,"us.census.acs.B16001001_quantile":null,"us.census.acs.B16001002_quantile":null,"us.census.acs.B16001003_quantile":null,"us.census.acs.B17001001_quantile":null,"us.census.acs.B17001002_quantile":null,"us.census.acs.B19013001_quantile":null,"us.census.acs.B19083001_quantile":null,"us.census.acs.B19301001_quantile":null,"us.census.acs.B25001001_quantile":null,"us.census.acs.B25002003_quantile":null,"us.census.acs.B25004002_quantile":null,"us.census.acs.B25004004_quantile":null,"us.census.acs.B25058001_quantile":null,"us.census.acs.B25071001_quantile":null,"us.census.acs.B25075001_quantile":null,"us.census.acs.B25075025_quantile":null}' as null_island_segmentation; )::text is null as null_island_segmentation;
WITH result as (
SELECT array_agg(_OBS_GetCategories) as expected FROM
cdb_observatory._OBS_GetCategories(
cdb_observatory._TestPoint(),
Array['us.census.spielman_singleton_segments.X10'],
'us.census.tiger.census_tract'
)
)
select (expected)[1]::text = '{"category":"Wealthy, urban without Kids","name":"Spielman-Singleton Segments: 10 Clusters","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 10 clusters"}' as GetCategories_at_test_point_1
from result;
WITH result as (
SELECT array_agg(_OBS_GetCategories) as expected FROM
cdb_observatory._OBS_GetCategories(
ST_SetSRID(ST_Point(0,0), 4326),
Array['us.census.spielman_singleton_segments.X10'],
'us.census.tiger.census_tract'
)
)
select expected[0] is NULL as GetCategories_at_null_island
from result;
-- Point-based OBS_GetMeasure with zillow -- Point-based OBS_GetMeasure with zillow
SELECT abs(OBS_GetMeasure_zhvi_point - 597900) / 597900 < 5.0 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure( SELECT abs(OBS_GetMeasure_zhvi_point - 597900) / 597900 < 5.0 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure(
@ -158,7 +112,7 @@ SELECT cdb_observatory.OBS_GetCategory(
-- Poly-based OBS_GetCategory -- Poly-based OBS_GetCategory
SELECT cdb_observatory.OBS_GetCategory( SELECT cdb_observatory.OBS_GetCategory(
cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Wealthy, urban without Kids' As obs_getcategory_polygon; cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Hispanic and Young' As obs_getcategory_polygon;
-- NULL Input OBS_GetCategory -- NULL Input OBS_GetCategory
SELECT cdb_observatory.OBS_GetCategory( SELECT cdb_observatory.OBS_GetCategory(
@ -215,7 +169,7 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory(
-- Area-based OBS_GetUSCensusCategory -- Area-based OBS_GetUSCensusCategory
SELECT cdb_observatory.OBS_GetUSCensusCategory( SELECT cdb_observatory.OBS_GetUSCensusCategory(
cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Wealthy, urban without Kids' As OBS_GetUSCensusCategory_polygon; cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Hispanic and Young' As OBS_GetUSCensusCategory_polygon;
-- Null-input OBS_GetUSCensusCategory -- Null-input OBS_GetUSCensusCategory
SELECT cdb_observatory.OBS_GetUSCensusCategory( SELECT cdb_observatory.OBS_GetUSCensusCategory(
@ -262,3 +216,535 @@ SELECT cdb_observatory.OBS_GetMeasureById(
'us.census.tiger.block_group', 'us.census.tiger.block_group',
'2010 - 2014' '2010 - 2014'
) IS NULL As OBS_GetMeasureById_null_id; ) IS NULL As OBS_GetMeasureById_null_id;
-- OBS_GetMeta null/null
SELECT cdb_observatory.OBS_GetMeta(NULL, NULL) IS NULL
AS OBS_GetMeta_null_null_is_null;
-- OBS_GetMeta null/empty array
SELECT cdb_observatory.OBS_GetMeta(NULL, '[]') IS NULL
AS OBS_GetMeta_null_empty_is_null;
-- OBS_GetMeta nullisland/null
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0), NULL) IS NULL
AS OBS_GetMeta_nullisland_null_is_null;
-- OBS_GetMeta nullisland/empty array
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0), '[]') IS NULL
AS OBS_GetMeta_nullisland_empty_is_null;
-- OBS_GetMeta nullisland/us_measure data
SELECT cdb_observatory.OBS_GetMeta(ST_Point(0, 0),
'[{"numer_id": "us.census.acs.B01003001"}]') IS NULL
AS OBS_GetMeta_nullisland_us_measure_is_null;
-- OBS_GetMeta for point completes one partial measure with "best" metadata
-- with no denominator
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001"}]') meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01003001' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'total_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Total Population' numer_name,
(meta->0->>'denom_id') IS NULL denom_id,
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
(meta->0->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for point completes one partial measure with "best" metadata
-- with a denominator
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01001002"}]') meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Male Population' numer_name,
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
(meta->0->>'denom_type') = 'Numeric' denom_type,
(meta->0->>'denom_name') = 'Total Population' denom_name,
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
(meta->0->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for polygon completes one partial measure with "best" metadata
-- with no denominator
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001"}]') meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01003001' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'total_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Total Population' numer_name,
(meta->0->>'denom_id') IS NULL denom_id,
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
(meta->0->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for polygon completes one partial measure with "best" metadata
-- with a denominator
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01001002"}]') meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Male Population' numer_name,
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
(meta->0->>'denom_type') = 'Numeric' denom_type,
(meta->0->>'denom_name') = 'Total Population' denom_name,
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
(meta->0->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for point completes several partial measures with "best"
-- metadata, includes geom alternatives if asked
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01001002"}]', null, 2) meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Male Population' numer_name,
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
(meta->0->>'denom_type') = 'Numeric' denom_type,
(meta->0->>'denom_name') = 'Total Population' denom_name,
(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id,
(meta->0->>'normalization') IS NULL normalization,
(meta->1->>'id')::integer = 1 id,
(meta->1->>'numer_id') = 'us.census.acs.B01001002' numer_id,
(meta->1->>'timespan_rank')::integer = 1 timespan_rank,
(meta->1->>'score_rank')::integer = 2 score_rank,
(meta->1->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->1->>'numer_colname') = 'male_pop' numer_colname,
(meta->1->>'numer_type') = 'Numeric' numer_type,
(meta->1->>'numer_name') = 'Male Population' numer_name,
(meta->1->>'denom_id') = 'us.census.acs.B01003001' denom_id,
(meta->1->>'denom_aggregate') = 'sum' denom_aggregate,
(meta->1->>'denom_colname') = 'total_pop' denom_colname,
(meta->1->>'denom_type') = 'Numeric' denom_type,
(meta->1->>'denom_name') = 'Total Population' denom_name,
(meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
(meta->1->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for point completes several partial measures with "best" metadata
-- with pre-computed geom
WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01001002", "geom_id": "us.census.tiger.census_tract"}]') meta)
SELECT
(meta->0->>'id')::integer = 1 id,
(meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id,
(meta->0->>'timespan_rank')::integer = 1 timespan_rank,
(meta->0->>'score_rank')::integer = 1 score_rank,
(meta->0->>'numer_aggregate') = 'sum' numer_aggregate,
(meta->0->>'numer_colname') = 'male_pop' numer_colname,
(meta->0->>'numer_type') = 'Numeric' numer_type,
(meta->0->>'numer_name') = 'Male Population' numer_name,
(meta->0->>'denom_id') = 'us.census.acs.B01003001' denom_id,
(meta->0->>'denom_aggregate') = 'sum' denom_aggregate,
(meta->0->>'denom_colname') = 'total_pop' denom_colname,
(meta->0->>'denom_type') = 'Numeric' denom_type,
(meta->0->>'denom_name') = 'Total Population' denom_name,
(meta->0->>'geom_id') = 'us.census.tiger.census_tract' geom_id,
(meta->0->>'normalization') IS NULL normalization
FROM meta;
-- OBS_GetMeta for point completes several partial measures with conflicting
-- metadata
SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01001002", "denom_id": "us.census.acs.B01001002", "geom_id": "us.census.tiger.census_tract"}]') IS NULL
AS obs_getmeta_conflicting_metadata;
-- OBS_GetData/OBS_GetMeta by id with empty list/null
WITH data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::TEXT[], null))
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_geomval_empty_null FROM data;
-- OBS_GetData/OBS_GetMeta by geom with empty list/null
WITH data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::GEOMVAL[], null))
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_text_empty_null FROM data;
-- OBS_GetData/OBS_GetMeta by geom with empty list
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(ARRAY[]::GEOMVAL[],
(SELECT meta FROM meta)))
SELECT ARRAY_AGG(data) IS NULL AS obs_getdata_geomval_empty_one_measure FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure NULL
-- normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 10923) / 10923 < 0.001 data_point_measure_null,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure NULL
-- normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_null,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure area
-- normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 10923) / 10923 < 0.001 data_point_measure_area,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure area
-- normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_area,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with one standard measure predenom
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 1900) / 1900 < 0.001 data_point_measure_predenominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one standard measure predenom
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 12327) / 12327 < 0.001 data_polygon_measure_predenominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with impossible denom
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "denominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
data->0->>'value' IS NULL data_point_measure_impossible_denominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one impossible denom
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "denominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
data->0->>'value' IS NULL data_polygon_measure_impossible_denominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with denom
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.acs.B01001002", "normalization": "denominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 0.6215) / 0.6215 < 0.001 data_point_measure_denominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one denom measure
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01001002", "normalization": "denominated"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_denominated,
data->1 IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with two standard measures NULL normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001"}, {"numer_id": "us.census.acs.B01001002"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_one_null,
abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with two standard measures predenom normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "predenom"}, {"numer_id": "us.census.acs.B01001002", "normalization": "predenom"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 12327) / 12327 < 0.001 data_polygon_measure_one_predenom,
abs((data->1->>'value')::Numeric - 6043) / 6043 < 0.001 data_polygon_measure_two_predenom
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with two standard measures area normalization
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "normalization": "area"}, {"numer_id": "us.census.acs.B01001002", "normalization": "area"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_one_area,
abs((data->1->>'value')::Numeric - 7739) / 7739 < 0.001 data_polygon_measure_two_area
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with two standard measures different geoms
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.census_tract"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
abs((data->0->>'value')::Numeric - 16960) / 16960 < 0.001 data_polygon_measure_tract,
abs((data->1->>'value')::Numeric - 15787) / 15787 < 0.001 data_polygon_measure_bg
FROM data;
-- OBS_GetData/OBS_GetMeta by point geom with one categorical
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestPoint(),
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestPoint(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
data->0->>'value' = 'Wealthy transplants displacing long-term local residents' data_point_categorical,
data->1->>'value' IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by polygon geom with one categorical
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
data->0->>'value' = 'Hispanic Black mix multilingual, high poverty, renters, uses public transport' data_poly_categorical,
data->1->>'value' IS NULL nullcol
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with one categorical and one measure
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"numer_id": "us.census.spielman_singleton_segments.X55"}, {"numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta)))
SELECT id = 1 id,
data->0->>'value' = 'Hispanic Black mix multilingual, high poverty, renters, uses public transport' data_poly_categorical,
abs((data->1->>'value')::Numeric - 15787) / 15787 < 0.0001 valcol
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.block_group"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta), false))
SELECT every(id = 1) is TRUE id,
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon + one measure
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta), false))
SELECT every(id = 1) is TRUE id,
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms,
abs(sum((data->1->>'value')::numeric) - 15787) / 15787 < 0.001 correct_pop
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with polygons inside a polygon + one measure + one text
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group"}, {"numer_id": "us.census.tiger.name", "geom_id": "us.census.tiger.block_group"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta), false))
SELECT every(id = 1) is TRUE id,
count(distinct (data->0->>'value')::geometry) = 16 correct_num_geoms,
abs(sum((data->1->>'value')::numeric) - 15787) / 15787 < 0.001 correct_pop,
array_agg(distinct data->2->>'value') = '{"Block Group 1","Block Group 2","Block Group 3","Block Group 4","Block Group 5"}' correct_bg_names
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with points inside a polygon
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.pointlm_geom"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta), false))
SELECT every(id = 1) AS id,
count(distinct (data->0->>'value')::geometry(point, 4326)) = 3 correct_num_points
FROM data;
-- OBS_GetData/OBS_GetMeta by geom with points inside a polygon + one text
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.pointlm_geom"}, {"geom_id": "us.census.tiger.pointlm_geom", "numer_id": "us.census.tiger.fullname"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
(SELECT meta FROM meta), false))
SELECT every(id = 1) AS id,
count(distinct (data->0->>'value')::geometry(point, 4326)) = 3 correct_num_points,
array_agg(data->1->>'value') = '{"Bushwick Yards","Edward Block Square","Bushwick Houses"}' pointgeom_names
FROM data;
-- OBS_GetData by id with one standard measure
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY['36047048500'],
(SELECT meta FROM meta)))
SELECT id = '36047048500' AS id,
(abs((data->0->>'value')::numeric) - 5578) / 5578 < 0.001 obs_getdata_by_id_one_measure_null
FROM data;
-- OBS_GetData by id with one standard measure, predenominated
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"normalization": "predenominated", "geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY['36047048500'],
(SELECT meta FROM meta)))
SELECT id = '36047048500' AS id,
(abs((data->0->>'value')::numeric) - 3241) / 3241 < 0.001 obs_getdata_by_id_one_measure_predenom
FROM data;
-- OBS_GetData/OBS_GetMeta by id with two standard measures
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01003001"}, {"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.acs.B01001002"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY['36047048500'],
(SELECT meta FROM meta)))
SELECT id = '36047048500' AS id,
(abs((data->0->>'value')::numeric) - 5578) / 5578 < 0.001 obs_getdata_by_id_one_measure_null,
(abs((data->1->>'value')::numeric) - 0.6053) / 0.6053 < 0.001 obs_getdata_by_id_two_measure_null
FROM data;
-- OBS_GetData/OBS_GetMeta by id with one categorical
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.census_tract", "numer_id": "us.census.spielman_singleton_segments.X55"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY['36047048500'],
(SELECT meta FROM meta)))
SELECT id = '36047048500' AS id,
data->0->>'value' = 'Wealthy transplants displacing long-term local residents' obs_getdata_by_id_categorical
FROM data;
-- OBS_GetData/OBS_GetMeta by id with one geometry
WITH
meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
'[{"geom_id": "us.census.tiger.census_tract"}]') meta),
data AS (SELECT * FROM cdb_observatory.OBS_GetData(
ARRAY['36047048500'],
(SELECT meta FROM meta)))
SELECT id = '36047048500' AS id,
ST_GeometryType((data->0->>'value')::geometry) = 'ST_MultiPolygon' obs_getdata_by_id_geometry
FROM data;
-- OBS_GetData with an API + geomvals, no args
SELECT ARRAY['us.census.tiger.census_tract'] <@ array_agg(data->0->>'value') AS OBS_GetData_API_geomvals_no_args
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
'[{"numer_type": "text", "numer_colname": "boundary_id", "api_method": "obs_getavailableboundaries", "geom_geomref_colname": "boundary_id"}]',
false);
-- OBS_GetData with an API + geomvals, args, numeric
SELECT json_typeof(data->0->'value') = 'number' AS OBS_GetData_API_geomvals_args_numer_return
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
'[{"numer_type": "numeric", "numer_colname": "obs_getmeasure", "api_method": "obs_getmeasure", "api_args": ["us.census.acs.B01003001"]}]', false);
-- OBS_GetData with an API + geomvals, args, text
SELECT json_typeof(data->0->'value') = 'string' AS OBS_GetData_API_geomvals_args_string_return
FROM cdb_observatory.obs_getdata(array[(cdb_observatory._testarea(), 1)::geomval],
'[{"numer_type": "text", "numer_colname": "obs_getcategory", "api_method": "obs_getcategory", "api_args": ["us.census.spielman_singleton_segments.X55"]}]', false);
-- OBS_GetData with an API + geomrefs, args, numeric
SELECT json_typeof(data->0->'value') = 'number' AS OBS_GetData_API_geomrefs_args_numer_return
FROM cdb_observatory.obs_getdata(array['36047076200'],
'[{"numer_type": "numeric", "numer_colname": "obs_getmeasurebyid", "api_method": "obs_getmeasurebyid", "api_args": ["us.census.acs.B01003001", "us.census.tiger.census_tract"]}]');
-- OBS_GetData with an API + geomrefs, args, text
SELECT json_typeof(data->0->'value') = 'string' AS OBS_GetData_API_geomrefs_args_string_return
FROM cdb_observatory.obs_getdata(array['36047'],
'[{"numer_type": "text", "numer_colname": "obs_getboundarybyid", "api_method": "obs_getboundarybyid", "api_args": ["us.census.tiger.county"]}]');

24
src/pg/test/sql/42_observatory_exploration_test.sql
View File

@ -10,11 +10,11 @@ SET client_min_messages TO WARNING;
-- _OBS_SearchTables tests -- _OBS_SearchTables tests
SELECT SELECT
t.table_name = 'obs_1babf5a26a1ecda5fb74963e88408f71d0364b81' As _OBS_SearchTables_tables_match, t.table_name = 'obs_0310c639744a2014bb1af82709228f05b59e7d3d' As _OBS_SearchTables_tables_match,
t.timespan = '2014' As _OBS_SearchTables_timespan_matches t.timespan = '2015' As _OBS_SearchTables_timespan_matches
FROM cdb_observatory._OBS_SearchTables( FROM cdb_observatory._OBS_SearchTables(
'us.census.tiger.county', 'us.census.tiger.county',
'2014' '2015'
) As t(table_name, timespan); ) As t(table_name, timespan);
-- _OBS_SearchTables tests -- _OBS_SearchTables tests
@ -362,7 +362,7 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
SELECT ARRAY_AGG(column_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5'] 'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_5km_buffer AS _obs_geometryscores_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
@ -382,7 +382,7 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
SELECT ARRAY_AGG(column_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY[ 'us.census.tiger.zcta5', 'us.census.tiger.census_tract', ARRAY[ 'us.census.tiger.zcta5', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.block_group'] 'us.census.tiger.county', 'us.census.tiger.block_group' ]
AS _obs_geometryscores_500km_buffer AS _obs_geometryscores_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
@ -410,7 +410,7 @@ SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text
WHERE table_id LIKE '%2015%'; WHERE table_id LIKE '%2015%';
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }' '{ "us.census.tiger.block_group" : 880, "us.census.tiger.census_tract" : 310, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
AS _obs_geometryscores_numgeoms_5km_buffer AS _obs_geometryscores_numgeoms_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
@ -419,7 +419,7 @@ SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
WHERE table_id LIKE '%2015%'; WHERE table_id LIKE '%2015%';
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }' '{ "us.census.tiger.block_group" : 11531, "us.census.tiger.census_tract" : 3601, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }'
AS _obs_geometryscores_numgeoms_50km_buffer AS _obs_geometryscores_numgeoms_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
@ -428,7 +428,7 @@ SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
WHERE table_id LIKE '%2015%'; WHERE table_id LIKE '%2015%';
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 48420, "us.census.tiger.census_tract" : 15775, "us.census.tiger.zcta5" : 6533, "us.census.tiger.county" : 304 }' '{ "us.census.tiger.block_group" : 48917, "us.census.tiger.census_tract" : 15969, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 314 }'
AS _obs_geometryscores_numgeoms_500km_buffer AS _obs_geometryscores_numgeoms_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
@ -437,7 +437,7 @@ SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
WHERE table_id LIKE '%2015%'; WHERE table_id LIKE '%2015%';
SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text = SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 169119, "us.census.tiger.census_tract" : 56380, "us.census.tiger.zcta5" : 26498, "us.census.tiger.county" : 2704 }' '{ "us.census.tiger.block_group" : 169191, "us.census.tiger.census_tract" : 56469, "us.census.tiger.zcta5" : 26525, "us.census.tiger.county" : 2753 }'
AS _obs_geometryscores_numgeoms_2500km_buffer AS _obs_geometryscores_numgeoms_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
@ -446,8 +446,8 @@ SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
WHERE table_id LIKE '%2015%'; WHERE table_id LIKE '%2015%';
SELECT ARRAY_AGG(column_id ORDER BY score DESC) = SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.census_tract', ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.zcta5', 'us.census.tiger.block_group'] 'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_50_geoms AS _obs_geometryscores_500km_buffer_50_geoms
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
@ -457,7 +457,7 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
SELECT ARRAY_AGG(column_id ORDER BY score DESC) SELECT ARRAY_AGG(column_id ORDER BY score DESC)
= ARRAY['us.census.tiger.zcta5', 'us.census.tiger.census_tract', = ARRAY['us.census.tiger.zcta5', 'us.census.tiger.census_tract',
'us.census.tiger.block_group', 'us.census.tiger.county'] 'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_500_geoms AS _obs_geometryscores_500km_buffer_500_geoms
FROM cdb_observatory._OBS_GetGeometryScores( FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),

34
src/pg/test/sql/44_observatory_geometries_test.sql
View File

@ -42,7 +42,7 @@ SELECT cdb_observatory.OBS_GetBoundary(
SELECT cdb_observatory.OBS_GetBoundary( SELECT cdb_observatory.OBS_GetBoundary(
cdb_observatory._TestPoint(), cdb_observatory._TestPoint(),
'us.census.tiger.census_tract', 'us.census.tiger.census_tract',
'2014' '2015'
) = :'cartodb_census_tract_geometry' As OBS_GetBoundary_year_census_tract; ) = :'cartodb_census_tract_geometry' As OBS_GetBoundary_year_census_tract;
-- should return null -- should return null
@ -65,7 +65,7 @@ SELECT cdb_observatory.OBS_GetBoundaryId(
SELECT cdb_observatory.OBS_GetBoundaryId( SELECT cdb_observatory.OBS_GetBoundaryId(
cdb_observatory._TestPoint(), cdb_observatory._TestPoint(),
'us.census.tiger.census_tract', 'us.census.tiger.census_tract',
'2014' '2015'
) = '36047048500'::text As OBS_GetBoundaryId_cartodb_census_tract_with_year; ) = '36047048500'::text As OBS_GetBoundaryId_cartodb_census_tract_with_year;
-- should give back '36047', the geoid of cartodb's county (King's/ -- should give back '36047', the geoid of cartodb's county (King's/
@ -104,7 +104,7 @@ SELECT cdb_observatory.OBS_GetBoundaryById(
SELECT cdb_observatory.OBS_GetBoundaryById( SELECT cdb_observatory.OBS_GetBoundaryById(
'36047', '36047',
'us.census.tiger.county', 'us.census.tiger.county',
'2014' '2015'
) = :'cartodb_county_geometry' OBS_GetBoundaryById_boundary_id_mismatch_geom_id; ) = :'cartodb_county_geometry' OBS_GetBoundaryById_boundary_id_mismatch_geom_id;
-- should give null since boundary_id does not match geometry reference id -- should give null since boundary_id does not match geometry reference id
@ -115,8 +115,7 @@ SELECT cdb_observatory.OBS_GetBoundaryById(
-- _OBS_GetBoundariesByGeometry -- _OBS_GetBoundariesByGeometry
SELECT SELECT array_agg(geom_refs) = Array[ '1104486618765', '1104486642837',
array_agg(geom_refs) = Array[ '1104486618765', '1104486642837',
'1104991798384', '1105044325367', '1104991798384', '1105044325367',
'1105089330200', '1105089331758'] '1105089330200', '1105089331758']
As _OBS_GetBoundariesByGeometry_roads_around_cartodb As _OBS_GetBoundariesByGeometry_roads_around_cartodb
@ -126,7 +125,7 @@ FROM (
-- near CartoDB's office -- near CartoDB's office
ST_MakeEnvelope(-74,40.69,-73.99,40.7, ST_MakeEnvelope(-74,40.69,-73.99,40.7,
4326), 4326),
'us.census.tiger.prisecroads') 'us.census.tiger.prisecroads_geom')
ORDER BY geom_refs ASC ORDER BY geom_refs ASC
) As m(the_geom, geom_refs); ) As m(the_geom, geom_refs);
@ -140,7 +139,7 @@ FROM (
-- near CartoDB's office -- near CartoDB's office
ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524, ST_MakeEnvelope(-73.9452409744,40.6988851644,-73.9280319214,40.7101254524,
4326), 4326),
'us.census.tiger.pointlm') 'us.census.tiger.pointlm_geom')
ORDER BY geom_refs ASC ORDER BY geom_refs ASC
) As m(the_geom, geom_refs); ) As m(the_geom, geom_refs);
@ -299,7 +298,7 @@ FROM (
-73.9280319214,40.7101254524, -73.9280319214,40.7101254524,
4326), 4326),
'us.census.tiger.census_tract', 'us.census.tiger.census_tract',
'2014') '2015')
ORDER BY geom_refs ASC ORDER BY geom_refs ASC
) As m(the_geom, geom_refs); ) As m(the_geom, geom_refs);
@ -342,7 +341,7 @@ FROM (
cdb_observatory._testpoint(), cdb_observatory._testpoint(),
500, 500,
'us.census.tiger.census_tract', 'us.census.tiger.census_tract',
'2014') '2015')
ORDER BY geom_refs ASC ORDER BY geom_refs ASC
) As m(the_geom, geom_refs); ) As m(the_geom, geom_refs);
@ -359,21 +358,4 @@ FROM (
ORDER BY geom_refs ASC ORDER BY geom_refs ASC
) As m(the_geom, geom_refs); ) As m(the_geom, geom_refs);
-- _OBS_GetGeometryMetadata
-- get metadata for census tracts
SELECT
geoid_colname = 'geoid' As geoid_name_matches,
target_table = 'obs_87a814e485deabe3b12545a537f693d16ca702c2' As table_name_matches,
geom_colname = 'the_geom' As geom_name_matches
FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract')
As m(geoid_colname, target_table, geom_colname);
-- get metadata for boundaries with clipped geometries
SELECT
geoid_colname = 'geoid' As geoid_name_matches,
target_table = 'obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963' As table_name_matches,
geom_colname = 'the_geom' As geom_name_matches
FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract_clipped') As m(geoid_colname, target_table, geom_colname);
\i test/fixtures/drop_fixtures.sql \i test/fixtures/drop_fixtures.sql

9
src/python/Makefile Normal file
View File

@ -0,0 +1,9 @@
include ../../Makefile.global
# Install the package locally for development
install:
pip install --upgrade ./observatory
test:
#TODO noop

0
src/python/observatory/observatory/__init__.py Normal file
View File

11
src/python/observatory/observatory/osm.py Normal file
View File

@ -0,0 +1,11 @@
from overpass import API
import json
def get_overpass(query):
'''
Return results of a raw overpass query.
'''
api = API()
response = api.Get(query)
return response['features']

1
src/python/observatory/requirements.txt Normal file
View File

@ -0,0 +1 @@
overpass==0.5.6

50
src/python/observatory/setup.py Normal file
View File

@ -0,0 +1,50 @@
"""
CartoDB Spatial Analysis Python Library
See:
https://github.com/CartoDB/crankshaft
"""
from setuptools import setup, find_packages
setup(
name='observatory',
version='0.0.1',
description='CARTO Observatory Python Library',
url='https://github.com/CartoDB/observatory-extension',
author='Research and Data - CARTO',
author_email='john@carto.com',
license='MIT',
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps mapping tools spatial data',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest', 'nose', 'mock'],
},
# The choice of component versions is dictated by what's
# provisioned in the production servers.
# IMPORTANT NOTE: please don't change this line. Instead issue a ticket to systems for evaluation.
install_requires=[],
#install_requires=['overpass==0.5.6'],
requires=[],
#requires=['overpass'],
#test_suite='test'
)

12
src/python/test/perftest.py
View File

@ -295,11 +295,11 @@ def test_getmeasure_split_performance(geom_complexity, normalization, geom, boun
for rows in rownums: for rows in rownums:
stmt = ''' stmt = '''
with data as ( with data as (
SELECT * FROM {schema}{api_method}datamulti( SELECT id, data FROM {schema}OBS_GetData(
(SELECT array_agg(({geom}, cartodb_id)::geomval) (SELECT array_agg(({geom}, cartodb_id)::geomval)
FROM obs_perftest_{complexity} FROM obs_perftest_{complexity}
WHERE cartodb_id <= {n}), WHERE cartodb_id <= {n}),
(SELECT {schema}{api_method}metamulti( (SELECT {schema}OBS_GetMeta(
(SELECT st_setsrid(st_extent({geom}), 4326) (SELECT st_setsrid(st_extent({geom}), 4326)
FROM obs_perftest_{complexity} FROM obs_perftest_{complexity}
WHERE cartodb_id <= {n}), WHERE cartodb_id <= {n}),
@ -309,18 +309,16 @@ with data as (
"geom_id": {boundary} "geom_id": {boundary}
}}]'::JSON }}]'::JSON
)) ))
) ))
AS x(cartodb_id INTEGER, measure Numeric))
UPDATE obs_perftest_{complexity} UPDATE obs_perftest_{complexity}
SET measure = data.measure SET measure = (data->0->>'value')::Numeric
FROM data FROM data
WHERE obs_perftest_{complexity}.cartodb_id = data.cartodb_id WHERE obs_perftest_{complexity}.cartodb_id = data.id
; ;
'''.format( '''.format(
point_or_poly='point' if geom == 'point' else 'polygon', point_or_poly='point' if geom == 'point' else 'polygon',
complexity=geom_complexity, complexity=geom_complexity,
schema='cdb_observatory.' if USE_SCHEMA else '', schema='cdb_observatory.' if USE_SCHEMA else '',
api_method='obs_getmeasure',
normalization=normalization, normalization=normalization,
geom=geom, geom=geom,
boundary=boundary.replace("'", '"'), boundary=boundary.replace("'", '"'),